solution: metadata: id: barista title: Building Agentic apps with Firebase and Google Cloud description: Learn how to build an agentic coffee ordering app with Firebase and Google Cloud fallbackCta: title: Launch in Firebase Studio icon: firebase-studio url: https://idx.google.com/import?url=https%3A%2F%2Fgithub.com%2FFirebaseExtended%2Fsolution-ai-barista accentColor: foreground: "white" background: "#141e2a" sections: - type: watch youtubeId: X_1zsJfYLMA title: Building Agentic apps with Firebase and Google Cloud description: Learn how to build an agentic coffee ordering app with Firebase and Google Cloud. technologies: - icon: /external-assets/angular.svg label: Angular url: https://angular.dev/ - icon: /external-assets/firebase-genkit.svg label: Genkit url: https://firebase.google.com/docs/genkit - icon: /external-assets/vertex-ai.svg label: Vertex AI with Gemini Models url: https://cloud.google.com/vertex-ai - icon: /external-assets/cloud-run.svg label: Cloud Run url: https://cloud.google.com/run - icon: /external-assets/firebase-firestore.svg label: Cloud Firestore url: https://firebase.google.com/docs/firestore - icon: /external-assets/firebase-storage.svg label: Cloud Storage for Firebase url: https://firebase.google.com/docs/storage - type: explore mode: video subsections: - id: video-1 title: Order beverages with the agent description: | The user interacts with the `OrderingAgent` through a chat interface to order beverages. The agent uses tool calls to add or change drinks in the order. videoPath: external-assets/agentic-barista/explore/1-orderCreation.mp4 layout: sidebyside logs: - timestamp: 1250 summary: Sending chat message detail: 'message: "I''d like a regular Cappucino with whole milk."' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 2500 summary: Send message to Gemini detail: '"I''d like a regular Cappucino with whole milk."' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 4000 summary: '[Tool call]: Add Cappucino (whole milk)' detail: 'add_drink("Cappucino", ["whole milk"])' file: orderTools.ts lineNumber: 35 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 5300 summary: '[Tool call]: Get Drinks in the order' detail: get_order() file: orderTools.ts lineNumber: 71 type: log sequence: - vertexai - timestamp: 6300 summary: Return response from Gemini detail: '"Got it. One Cappucino with Whole milk. Anything else?"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - timestamp: 11450 summary: Sending chat message detail: 'message: "I also want a Latte with hazelnut and three shots"' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 13000 summary: Send message to Gemini detail: '"I also want a Latte with hazelnut and three shots."' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 14000 summary: '[Tool call]: Add Latte (hazelnut sweetener, three shots)' detail: 'add_drink("Latte", ["hazelnut sweetener", "three shots"])' file: orderTools.ts lineNumber: 5 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 16000 summary: '[Tool call]: Get drinks in the order' detail: get_order() file: orderTools.ts lineNumber: 71 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 17450 summary: Return response from Gemini detail: '"And what kind of milk for the Latte?"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - timestamp: 19300 summary: Sending chat message detail: 'message: "2% milk"' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 20500 summary: Send message to Gemini detail: '"2% milk"' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 21500 summary: '[Tool call]: Get drinks in the order' detail: get_order() file: orderTools.ts lineNumber: 71 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 22500 summary: '[Tool call]: Update Latte' detail: 'upate_drink(1, "Latte", ["2% milk", "hazelnut sweetener", "three shots"])' file: orderTools.ts lineNumber: 52 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 24000 summary: '[Tool call]: Get drinks in the order' detail: get_order() file: orderTools.ts lineNumber: 71 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 25000 summary: Return response from Gemini detail: '"Okay. One Cappucino with Whole milk and one Latte with 2% milk, hazelnut sweetener and Triple shots. Anything else?"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - timestamp: 30250 summary: Sending chat message detail: 'message: "Instead of the cappuccino, make it a Mocha and an espresso."' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 32500 summary: Send message to Gemini detail: '"Instead of the cappuccino, make it a Mocha and an espresso."' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 33500 summary: '[Tool call]: Get Drinks in the order' detail: get_order() file: orderTools.ts lineNumber: 71 type: log sequence: - vertexai - timestamp: 35000 summary: '[Tool call]: Remove Cappucino' detail: remove_item(0) file: orderTools.ts lineNumber: 85 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 36500 summary: '[Tool call]: Add Mocha' detail: 'add_drink("Mocha", ["whole milk"])' file: orderTools.ts lineNumber: 35 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 38500 summary: '[Tool call]: Add Espresso' detail: add_drink("Espresso") file: orderTools.ts lineNumber: 35 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 40500 summary: '[Tool call]: Get Drinks in the order' detail: get_order() file: orderTools.ts lineNumber: 71 type: log sequence: - vertexai - timestamp: 41550 summary: Return response from Gemini detail: '"Okay. your order is now: One Latte with 2% milk, hazelnut sweetener and Triple shots, one Mocha with Whole Milk and one Espresso. Anything else?"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - id: video-2 title: Recommendations from multimodal input description: | The user can upload an image to receive a beverage recommendation. The media file is uploaded to Cloud Storage for Firebase from the Angular frontend and accessed directly through the Vertex AI plugin in Genkit on the backend. videoPath: external-assets/agentic-barista/explore/2-multiModal.mp4 layout: sidebyside logs: - timestamp: 3250 summary: Upload photo to Cloud Storage detail: 'Returned URL: "gs://barista-agent/user1/pH5a/media/342.jpg"' imageData: external-assets/agentic-barista/explore/beach.jpg sequence: - angular - storage inspect: angular-firebase - timestamp: 7250 summary: Sending chat message detail: '{message: "Here''s a photo from my last holiday. I want a drink that takes me back there.", media: {url: "gs://barista-agent/user1/pH5a/media/342.jpg", "contentType":"image/jpeg"}}' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 8000 summary: Send message to Gemini detail: '{message: "Here''s a photo from my last holiday. I want a drink that takes me back there.", media: {url: "gs://barista-agent/user1/pH5a/media/342.jpg", "contentType":"image/jpeg"}}' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 9000 summary: '[OrderingAgent] call [RecommendationAgent] for recommendation' detail: "" file: orderingAgent.ts lineNumber: 23 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 10000 summary: '[Tool call] RecommendationAgent' detail: '{message: "Here''s a photo from my last holiday. I want a drink that takes me back there.", media: {url: "gs://barista-agent/user1/pH5a/media/342.jpg", "contentType":"image/jpeg"}}' file: recommendationAgent.ts lineNumber: 21 type: log sequence: - vertexai inspect: angular-firebase - timestamp: 11500 summary: Read image from Cloud Storage detail: '"gs://barista-agent/user1/pH5a/media/342.jpg"' imageData: external-assets/agentic-barista/explore/beach.jpg sequence: - vertexai - genkit - storage inspect: angular-firebase - timestamp: 13250 summary: Return response from Gemini detail: '"Given the beautiful beach photo, I recommend an Iced Cold Brew or an Iced Latte with almond milk. If you prefer tea, try an Iced Green Tea, these would all be refreshing choices!"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - id: video-3 title: External tool calling description: | When asking for the recommended coffee of the day, the `RecommendationAgent` consults a tool that makes a call to an external HTTP API. videoPath: external-assets/agentic-barista/explore/3-recommendation.mp4 layout: sidebyside logs: - timestamp: 500 summary: Sending chat message detail: '"What is the recommended coffee of the day?"' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 1500 summary: Send message to Gemini detail: '"What is the recommended coffee of the day?"' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 2500 summary: '[OrderingAgent] call [RecommendationAgent] for recommendation' detail: "" file: orderingAgent.ts lineNumber: 23 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 3500 summary: '[Tool call] get_barista_recommendation' detail: "" file: recommendationAgent.ts lineNumber: 21 type: log sequence: - vertexai inspect: gemini-vertexai/gemini-vertexai-4 - timestamp: 4000 summary: HTTP request to external API detail: "" file: recommendationTools.ts lineNumber: 39 type: log sequence: - vertexai inspect: gemini-vertexai/gemini-vertexai-4 - timestamp: 5000 summary: HTTP response detail: '{"name":"Latte","modifiers":["double shots","chocolate syrup"]}' file: recommendationTools.ts lineNumber: 50 type: log sequence: - vertexai inspect: gemini-vertexai/gemini-vertexai-4 - timestamp: 5700 summary: Return response from Gemini detail: '"Today''s recommendation is a Latte with double shots and chocolate syrup. Would you like to try that?"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - id: video-4 title: Human-in-the-loop for order submission description: | Submitting a beverage order is a critical task that requires final confirmation from the user. The tool call that submits the order is interrupted and paused to request user confirmation before proceeding. videoPath: external-assets/agentic-barista/explore/4-orderSubmission.mp4 layout: sidebyside logs: - timestamp: 1900 summary: Sending chat message detail: 'message: "Submit my order"' file: chat.service.ts lineNumber: 80 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 2500 summary: Send message to Gemini detail: '"Submit my order"' file: agentSessionManager.ts lineNumber: 94 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 3000 summary: '[Tool call]: Submit order' detail: submit_order() file: orderTools.ts lineNumber: 114 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 4000 summary: '[Tool call]: INTERRUPT for user input' detail: "" file: orderTools.ts lineNumber: 129 type: warning sequence: - vertexai - timestamp: 5100 summary: Request approval from user detail: 'readyForSubmission: true, order: (...)' file: agentSessionManager.ts lineNumber: 103 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - timestamp: 7000 summary: Sending user approval detail: 'approved: true' file: chat.service.ts lineNumber: 65 type: log sequence: - angular - cloudrun inspect: genkit-production - timestamp: 8000 summary: Resuming execution detail: "" file: agentSessionManager.ts lineNumber: 55 type: log sequence: - genkit - vertexai inspect: gemini-vertexai - timestamp: 9500 summary: '[Tool call]: Resume submit order' detail: 'submit_order(resumed: true, approved: true)' file: orderTools.ts lineNumber: 135 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 11500 summary: '[Tool call]: Store order in Firestore' detail: "" file: orderTools.ts lineNumber: 146 type: log sequence: - vertexai inspect: gemini-vertexai - timestamp: 13000 summary: Return response from Gemini detail: '"Your oder has been submitted. Collect it under ''Outstanding Ostrich'' from the barista"' file: chat.ts lineNumber: 41 type: log sequence: - genkit - cloudrun - angular inspect: genkit-production - type: inspect subsections: - id: gemini-vertexai title: Agents with Gemini and Vertex AI examples: - title: Building agentic apps with Genkit and Gemini in Vertex AI icons: - /external-assets/firebase-genkit.svg - /external-assets/vertex-ai.svg mode: markdown info: | Tool calling and support for multimodal media inputs are essential for an agentic application like an interactive coffee ordering system. These features are supported by the Gemini models available on Vertex AI. The AI Barista app uses the Genkit AI toolkit to call Vertex AI. Learn more about building with Genkit and how we are using it to simplify integrating with AI calls in the following sections. To get started, set up the Vertex AI plugin in Genkit and select a model. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/genkit.ts - label: Vertex AI Plugin in Genkit url: https://firebase.google.com/docs/genkit/plugins/vertex-ai code: language: typescript file: external-assets/agentic-barista/inspect/gemini-vertexai/genkit-setup.ts - title: Designing effective agent prompts icons: - /external-assets/firebase-genkit.svg - /external-assets/vertex-ai.svg mode: markdown info: | Consider the main agent's overall goal, purpose, voice, and personality when designing its prompt. The agent interacts with the environment through carefully defined tools and functions that it calls to complete tasks. Providing examples in the prompt that include tool calls, expected inputs and outputs can improve the quality of the agent's responses. The `orderingAgent` definition includes a list of tools that are available. These tools are described in the prompt and included in all interaction examples. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/agents/orderingAgent/orderingAgent.ts code: language: typescript file: external-assets/agentic-barista/inspect/gemini-vertexai/agent-prompts.ts - title: Using specialized agents icons: - /external-assets/firebase-genkit.svg - /external-assets/vertex-ai.svg mode: markdown info: | Breaking down your application into multiple agents enables each of them to specialize on particular tasks. This simplifies testing and debugging prompts, as well as structuring the interaction with other tools and systems. The `recommendationAgent` specializes in providing beverage recommendations. It accepts multimodal input and interacts directly with the user for any follow-up questions. The `orderingAgent` references the `recommendationAgent` as a tool and calls it when required. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/prompts/recommendationAgent.prompt code: language: text file: external-assets/agentic-barista/inspect/gemini-vertexai/recommendationAgentPrompt.txt - title: Building agent tools with Genkit icons: - /external-assets/firebase-genkit.svg mode: markdown info: | Tool calling (or function calling) gives the model a structured way to interact with its environment. A tool consists of input parameters (provided by the model), a function description and expected output. The model calls these functions to fulfill its objective and complete its task. Custom data can be stored in a Genkit session that is available within a tool call and can be directly modified during its execution. In the AI Barista, the user's drink order is stored in the session and updated by tools that add, remove or update items. This session-internal storage speeds up execution and ensures constant availability for frequent access, such as when grounding responses. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/agents/orderingAgent/orderTools.ts - label: Persistent chat sessions in Genkit url: https://firebase.google.com/docs/genkit/chat#stateful_chat_sessions code: language: typescript file: external-assets/agentic-barista/inspect/gemini-vertexai/tools-genkit.ts - title: Connecting agent tools to APIs icons: - /external-assets/firebase-genkit.svg mode: markdown info: | Tools can call external services to fulfill requests. For example, accessing an external service to send a message, look up data or store information. The `RecommendationAgent` uses a tool to get a beverage recommendation from the barista. This recommendation is available from a HTTP endpoint that that the `get_barista_recommendation` tool accesses. The tool processes the response and returns the recommendation to the agent. It also includes a fallback mechanism in case the service is unavailable. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/agents/recommendationAgent/recommendationTools.ts code: language: typescript file: external-assets/agentic-barista/inspect/gemini-vertexai/tools-apis.ts - title: Implementing Human-in-the-loop interactions icons: - /external-assets/firebase-genkit.svg mode: markdown info: | Human-in-the-loop flows enable models to clarify information or confirm an action with the user before it is completed. The LLM generation is paused by the tool and resumed with the user's input. Submission of the final beverage order is a critical task that requires the user to explicitly confirm the action. In the tool `submit_order` execution is interrupted when user input is required. Once the user has responded, the tool call resumes and the order is submitted if it has been approved. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/agents/orderingAgent/orderTools.ts#L124 - label: Tool interrupts in Genkit url: https://firebase.google.com/docs/genkit/interrupts code: language: typescript file: external-assets/agentic-barista/inspect/gemini-vertexai/tools-hitl.ts - id: genkit-production title: Using Genkit in Production examples: - title: Building a conversational chat interface with Genkit icons: - /external-assets/firebase-genkit.svg mode: markdown info: | Genkit is a toolkit that provides functionality to build AI-powered features for your application. It includes abstractions that simplify building a conversation-based interface to interact with an AI model, including managing the history and conversational state across a user's entire interaction. The backend of the AI Barista app includes an HTTP endpoint that serves as the chat interface with the agent, making it accessible to the frontend and client apps. It wraps calls to Genkit that load the user's session and resume an on-going conversation with the `OrderingAgent`. links: - label: Chat sessions in Genkit url: https://firebase.google.com/docs/genkit/chat code: language: typescript file: external-assets/agentic-barista/inspect/genkit-production/chat-interface.ts - title: Deploying Genkit on Cloud Run icons: - /external-assets/firebase-genkit.svg - /external-assets/cloud-run.svg mode: markdown info: | You can deploy flows defined in Genkit directly to Cloud Functions for Firebase or any Node.js hosting platform. However, the AI Barista app uses Genkit in an Express framework application that provides additional functionality and context for each call, including managing users and persistent chat sessions. This Express app includes endpoints for a chat interface with the agent, responding to human-in-the-loop requests and for initializing a new session. The AI Barista NodeJS application can run locally or can be deployed to Google Cloud Run. Cloud Run is a managed compute platform that scales containers and can be deployed directly from source code. Applications on Cloud Run directly integrate with Google Cloud services, including Cloud Logging, Firestore and other Google Cloud and Firebase services. links: - label: Deploying Genkit Flows as web API endpoints url: https://firebase.google.com/docs/genkit/flows#deploying_flows - label: Deploying Cloud Run services from source code url: https://cloud.google.com/run/docs/deploying-source-code code: language: typescript file: external-assets/agentic-barista/inspect/genkit-production/deploy-genkit.ts - title: Securing tool execution with Firebase Authentication and agent context icons: - /external-assets/firebase-genkit.svg - /external-assets/firebase-auth.svg mode: markdown info: | Agents call tools to interact with their environment, which often requires additional information for their execution. This could include the user's account details, their saved preferences or other data that is not needed for the model response, but is required for tools to complete their tasks. Authentication information is crucial when calling tools as it governs access to user-specific resources. This should be provided in the _execution context_ to the tool call. Note that when deploying Genkit flows directly to a serverless environment on Firebase or Google Cloud, the authentication details are automatically included in the context during execution. In other cases, such as when calling Genkit through an Express framework application, the details need to be added manually. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/agentSessionManager.ts - label: Passing information through context in Genkit url: https://firebase.google.com/docs/genkit/context code: language: typescript file: external-assets/agentic-barista/inspect/genkit-production/auth-context.ts - title: Monitoring Genkit performance icons: - /external-assets/firebase-genkit.svg - /external-assets/cloud-observability.svg mode: markdown info: | Genkit Monitoring enables monitoring and real-time insights into the application's performance by exporting key metrics and traces. This includes error rates, latency, token usage, and execution traces to help debug and improve the application. After enabling telemetry, Genkit sends these metrics to Google Cloud Observability. This lets you analyze and track the health of the application remotely on the Genkit Monitoring dashboard and the Google Cloud Logging viewer. links: - label: Get started with Genkit Monitoring url: https://firebase.google.com/docs/genkit/observability/getting-started - label: Learn about telemetry in Genkit url: https://firebase.google.com/docs/genkit/observability/telemetry-collection code: language: typescript file: external-assets/agentic-barista/inspect/genkit-production/telemetry.ts - id: cloudrun-firestore title: Cloud Run and Firestore examples: - title: Storing drink orders in Firestore with the Firebase Admin SDK icons: - /external-assets/firebase-firestore.svg mode: markdown info: | The agent stores final drink orders in a dedicated collection in Firestore, ready to be picked up by another service. When running in Google environments, such as Cloud Run or Cloud Functions, the [Firebase Admin SDK](https://firebase.google.com/docs/admin/setup) can directly interact with Firebase. It supports key Firebase services, including Firebase Authentication, Cloud Firestore and other Google Cloud services. The AI Barista app's backend is deployed to Cloud Run and uses the Firebase Admin SDK to store final beverage orders in Cloud Firestore. It stores each order in a new document that includes the user's ID from Firebase Authentication in its path to manage access and accommodate multiple orders. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/firestore/submittedOrderStore.ts - label: Learn more about Cloud Firestore url: https://firebase.google.com/docs/firestore - label: Get started with the Firebase Admin SDK url: https://firebase.google.com/docs/admin/setup code: language: typescript file: external-assets/agentic-barista/inspect/cloudrun-firestore/storing-orders.ts - title: Persisting agent sessions in Cloud Firestore icons: - /external-assets/firebase-genkit.svg - /external-assets/firebase-firestore.svg mode: markdown info: | Chat and agent-based interaction flows require persistent storage to save and restore context when deployed to serverless environments. This enables the resumption of an on-going agent session to continue task or the conversation. The AI Barista app stores this context in Cloud Firestore, tied to the user's authenticated identity from Firebase Authentication. Cloud Firestore is designed to scale and integrate with Firebase applications. The `SesssionStore` interface describes how to design persistent storage for session state and context for Genkit. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/firestore/sessionStore.ts - label: Session persistence in Genkit url: https://firebase.google.com/docs/genkit/chat#session_persistence_experimental code: language: typescript file: external-assets/agentic-barista/inspect/cloudrun-firestore/persistent-sessions.ts - id: cloudstorage-genkit title: Multimodal input with Firebase Cloud Storage and Genkit examples: - title: Enabling multimodal media inputs with Gemini icons: - /external-assets/gemini.svg mode: markdown info: | Multimodal models process prompts that also include media such as images, video and audio, in addition to text. This includes the latest models from the Gemini family that are recommended for building agentic experiences. These multimodal inputs can provide additional context and enable innovative interaction methods in your application. The AI Barista's `RecommendationAgent` suggests beverages based on the user's input, including multimodal media. For example, users can provide a photo from their last holiday or a favourite painting to receive a personalized beverage recommendation. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/prompts/recommendationAgent.prompt - label: Learn about designing multimodal prompts url: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/design-multimodal-prompts - label: Learn more about multimodal AI url: https://cloud.google.com/use-cases/multimodal-ai code: language: text file: /external-assets/agentic-barista/inspect/cloudstorage-genkit/recommendationAgentPrompt.txt - title: Uploading media from Angular to Cloud Storage for Firebase icons: - /external-assets/firebase-storage.svg - /external-assets/angular.svg mode: markdown info: | Cloud Storage for Firebase is a scalable object storage service that stores and serves user-generated content. It integrates with Firebase Authentication for built-in security and is available through SDKs that offer scalable and robust operations that simplify file handling and network access. The AI Barista uses Cloud Storage for Firebase for image handling and processing. The Angular app uploads images using the AngularFire library (including error handling and retries) which simplifies robust media handling in the frontend. The backend then references these images from Cloud Storage when calling the `RecommendationAgent` for a drink recommendation. Security is enforced through Firebase Authentication and Cloud Storage security rules, protecting and scoping user uploads. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/client/web/angular-customer-app/src/app/services/mediaStorage.service.ts - label: Get started with Cloud Storage for Firebase url: https://firebase.google.com/docs/storage/web/start - label: Get started with Firebase using AngularFire url: https://github.com/angular/angularfire?tab=readme-ov-file#angularfire code: language: typescript file: /external-assets/agentic-barista/inspect/cloudstorage-genkit/upload-angular.ts - title: Processing media from Cloud Storage in Genkit icons: - /external-assets/firebase-storage.svg - /external-assets/firebase-genkit.svg - /external-assets/vertex-ai.svg mode: markdown info: | The Vertex AI plugin for Genkit natively supports references to objects from Cloud Storage, eliminating the need to manually download media bytes. Instead of providing these media files in the prompt itself, the input only includes a reference to its location. This results in a smaller and more manageable agent context. The user uploads media and submits their message from the frontend. Next, the backend constructs a new chat message for the `OrderingAgent` in Genkit that contains the text and the Cloud Storage URL referencing the uploaded media. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/agentSessionManager.ts#L79 - label: Learn about the Vertex AI plugin in Genkit url: https://firebase.google.com/docs/genkit/plugins/vertex-ai code: language: typescript file: /external-assets/agentic-barista/inspect/cloudstorage-genkit/genkit-chat-media.ts - id: security-privacy title: Security, Safety and Privacy examples: - title: Prioritizing AI Safety icons: - /external-assets/gemini.svg - /external-assets/vertex-ai.svg mode: markdown info: | Address AI safety by considering your specific use case and potential user interactions. Gemini models support content filters and thresholds that can block potentially harmful responses based on your application's requirements. Also consider incorporating behavioral guidelines and guardrails into your system prompt to improve safety. For example, the prompt for the AI Barista guides the model to maintain a _friendly and helpful barista persona_, while restricting interactions to completing beverage orders. Services such as [Cloud Model Armor](https://cloud.google.com/security-command-center/docs/model-armor-overview) can also screen responses and prompts for security and safety risks. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/ai/genkit.ts - label: Learn more about Responsible AI url: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/responsible-ai - label: Safety and content filters with Vertex AI and Gemini models url: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/configure-safety-filters code: language: typescript file: external-assets/agentic-barista/inspect/security-privacy/safety-settings.ts - title: Securing Cloud Storage for Firebase with Security Rules icons: - /external-assets/firebase-storage.svg mode: markdown info: | Cloud Storage for Firebase integrates with Firebase Authentication to authorize users and validate requests. Firebase Security Rules control this access and control access to specific files, folders or actions. The AI Barista app stores authenticated users' media files in Cloud Storage for Firebase for drink recommendations. Firebase Security rules restrict access to these files to the user who uploaded them. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/firebase/storage.rules - label: Learn more about Firebase Security Rules for Cloud Storage url: https://firebase.google.com/docs/storage/security code: language: typescript file: external-assets/agentic-barista/inspect/security-privacy/storage-rules.txt - title: Securing Cloud Firestore with Security Rules icons: - /external-assets/firebase-firestore.svg mode: markdown info: | Cloud Firestore, like Cloud Storage for Firebase, uses Security Rules to provide access control and data validation. Firebase Authentication integrates with these rules to support user- and role-based access control. However, the AI Barista app accesses Cloud Firestore through the Firebase Admin SDK on Cloud Run, where access is managed through [Identity and Access Management (IAM) for Cloud Firestore](https://cloud.google.com/firestore/docs/security/iam) instead of security rules. This example shows how to structure security rules for the AI Barista Firestore database for different clients. Role-based access controls could be used to limit access to submitted orders only to baristas to mark an order as 'ready for pickup'. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/firebase/firestore.rules - label: Learn more about Cloud Firestore Security Rules url: https://firebase.google.com/docs/firestore/security/get-started - label: Learn more about Role-based access controls url: https://firebase.google.com/docs/firestore/solutions/role-based-access code: language: typescript file: external-assets/agentic-barista/inspect/security-privacy/firestore-rules.txt - title: Managing data retention for Cloud Firestore and Cloud Storage icons: - /external-assets/firebase-firestore.svg - /external-assets/firebase-storage.svg - /external-assets/firebase-functions.svg mode: markdown info: | Applications should have a data retention policy that includes removal of data after a user has completed their task or has deleted their account. The AI Barista automatically removes stale data after a set expiration time or when an account is deleted. In Cloud Firestore, this is configured through time to live (TTL) policies that automatically delete data if a user's session has not been updated recently. A function on Cloud Functions for Firebase, triggered when associated data is removed from Cloud Firestore, automatically deletes the corresponding media files from Cloud Storage. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/functions/src/index.ts - label: Learn more about managing data retention in Cloud Firestore url: https://firebase.google.com/docs/firestore/ttl - label: Explore the 'Delete User Data' extension for Firebase url: https://firebase.google.com/docs/extensions/official/delete-user-data code: language: typescript file: external-assets/agentic-barista/inspect/security-privacy/function-delete-media.ts - id: angular-firebase title: Angular and Firebase examples: - title: Signing in users with Firebase Authentication in Angular icons: - /external-assets/angular.svg - /external-assets/firebase-auth.svg mode: markdown info: | Firebase Authentication integrates with other Firebase services to authenticate users, including security rules that manage data access. The AngularFire library provides additional support for integrating the Firebase SDK into Angular applications, including Firebase Authentication. The AI Barista app's Angular frontend uses anonymous authentication to create temporary accounts for the duration of the user's session. This enables security rules for protecting data access and provides a seamless user experience. Accounts can be onboarded and linked to profiles and full user accounts in the future. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/client/web/angular-customer-app/src/app/services/login.service.ts - label: Get started with Firebase using AngularFire url: https://github.com/angular/angularfire?tab=readme-ov-file#angularfire - label: Learn more about anonymous authentication url: https://firebase.google.com/docs/auth/web/anonymous-auth code: language: typescript file: external-assets/agentic-barista/inspect/angular-firebase/auth-angular.ts - title: Verifying identities with Firebase Authentication on the backend icons: - /external-assets/cloud-run.svg - /external-assets/firebase-auth.svg mode: markdown info: | After the user has authenticated using Firebase Authentication on the frontend, backends can use the Firebase Admin SDK to verify the authenticity and integrity of the user's profile. The AI Barista Angular frontend includes the user's `ID token` in all requests. The backend decodes and verifies this token for every request using an Express framework middleware. Its `uid` is then used to identify the user when accessing any services, such as Cloud Firestore and Cloud Storage. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/auth/verifyAuthenticationMiddleware.ts - label: Learn more about verifying ID tokens on the backend url: https://firebase.google.com/docs/auth/admin/verify-id-tokens code: language: typescript file: external-assets/agentic-barista/inspect/angular-firebase/auth-backend.ts - title: Protecting backend access with Firebase App Check icons: - /external-assets/angular.svg - /external-assets/firebase-appcheck.svg mode: markdown info: | Firebase App Check protects app backends by preventing access from unauthorized clients. Attestation is like authentication for an application; it checks that requests come from an authenticated, untampered app. App Check integrates with other Firebase services, such as Firebase Authentication and Cloud Storage. For Angular, the AngularFire library integrates with Firebase App Check for attestation of the web applications using reCAPTCHA enterprise. App Check ensures that all requests for the AI Barista app come from the authorized frontend application. Each backend request includes an App Check Token that is decoded and verified before it is processed. Replay protection verifies the authenticity of critical requests, such as the final order confirmation. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/client/web/angular-customer-app/src/app/services/login.service.ts - label: Learn more about Firebase App Check url: https://firebase.google.com/docs/app-check - label: Learn more about reCAPTCHA Enterprise for web url: https://cloud.google.com/security/products/recaptcha - label: Get started with Firebase using AngularFire url: https://github.com/angular/angularfire?tab=readme-ov-file#angularfire code: language: typescript file: external-assets/agentic-barista/inspect/angular-firebase/appcheck-angular.ts - title: Validating App Check attestations on the backend icons: - /external-assets/cloud-run.svg - /external-assets/firebase-auth.svg mode: markdown info: | Clients should include App Check tokens in all backend and service requests. Some Google Cloud and Firebase services, such as Data Connect, Cloud Firestore and Cloud Storage, integrate with App Check to automatically validate incoming requests. Custom backends, such as the AI Barista app deployed to Cloud Run, can use the Firebase Admin SDK to explicitly verify App Check tokens to confirm that requests originate from from authorized and legitimate clients. links: - type: github url: https://github.com/FirebaseExtended/solution-ai-barista/blob/main/services/cloud-run/src/auth/verifyAppCheckMiddleware.ts - label: Learn more about verifying App Check tokens from a backend url: https://firebase.google.com/docs/app-check/custom-resource-backend code: language: typescript file: external-assets/agentic-barista/inspect/angular-firebase/appcheck-backend.ts - type: quiz questions: - title: What makes an agentic application powerful? answers: - answer: Strict adherence to predefined scripts - answer: Interacting with the environment and making decisions to fulfill objectives correct: true - answer: Displaying only static information - answer: Reliance on human intervention for every step - title: Which features are essential for an agentic application like an interactive coffee ordering system? answers: - answer: Multimodal media inputs - answer: Tool calling - answer: Context storage - answer: All of the above correct: true - title: What is the primary purpose of tool calling (or function calling) in agentic applications? answers: - answer: Displaying static information - answer: Providing a structured way for the model to interact with its environment correct: true - answer: Reducing the need for any input parameters - answer: Limiting the model's ability to complete tasks - title: What is the benefit of breaking an application into multiple specialized agents? answers: - answer: It increases overall application complexity. - answer: It requires each agent to have identical functionality. - answer: It eliminates all tool or function calls. - answer: It simplifies testing, debugging, and tool interaction. correct: true - title: What does Genkit help build? answers: - answer: Web apps with Angular - answer: AI-powered features correct: true - answer: Real-time, collaborative database services - answer: Monitoring for Cloud Firstore and Cloud Storage - title: How can multimodal input improve interactions with AI systems? answers: - answer: By providing richer and more natural ways of communication correct: true - answer: By limiting the system to only text-based responses - answer: By making the AI system's responses less accurate - answer: By removing the need for any user input - title: What is the purpose of enabling telemetry in Genkit? answers: - answer: Collecting and processing user inputs - answer: Monitoring AI performance metrics correct: true - answer: Blocking requests from unauthorized clients - answer: Analyzing frontend user interactions - title: What feature allows definition of access permissions for data in Cloud Storage for Firebase? answers: - answer: Firewall rules - answer: Firebase App Check - answer: Firebase Security Rules correct: true - answer: Client-side access control - type: build promoType: firebase-studio links: - url: https://github.com/FirebaseExtended/solution-ai-barista label: Open on GitHub - url: https://developers.google.com/solutions label: Explore more Solutions architecture: entities: - id: angular icon: /external-assets/angular.svg label: Angular x: 2 y: -2 connections: - from: angular to: appcheck inspect: angular-firebase/angular-firebase-2 - from: angular to: hosting - from: angular to: auth inspect: angular-firebase/angular-firebase-0 - from: angular to: cloudrun inspect: angular-firebase - from: angular to: storage inspect: cloudstorage-genkit/cloudstorage-genkit-1 - id: hosting icon: /external-assets/firebase-hosting.svg label: Firebase Hosting x: 2 y: 0 - id: auth icon: /external-assets/firebase-auth.svg label: Firebase Authentication x: 0 y: -2 - id: appcheck icon: /external-assets/firebase-appcheck.svg label: Firebase App Check x: 0 y: 0 - id: cloudrun icon: /external-assets/cloud-run.svg label: Cloud Run x: 4 y: -2 connections: - from: cloudrun to: firestore inspect: cloudrun-firestore - from: cloudrun to: genkit inspect: genkit-production - id: firestore icon: /external-assets/firebase-firestore.svg label: Cloud Firestore x: 4 y: -4 connections: - from: genkit to: firestore inspect: genkit-production - id: storage icon: /external-assets/firebase-storage.svg label: Cloud Storage for Firebase x: 4 y: 0 connections: - from: storage to: genkit inspect: cloudstorage-genkit/cloudstorage-genkit-2 - id: genkit icon: /external-assets/firebase-genkit.svg label: Genkit x: 6 y: -2 connections: - from: genkit to: vertexai inspect: gemini-vertexai - from: genkit to: logging inspect: genkit-production/genkit-production-3 - id: logging icon: /external-assets/cloud-observability.svg label: Cloud Logging x: 7 y: -4 - id: vertexai icon: /external-assets/vertex-ai.svg label: Vertex AI x: 7 y: 0 badges: startBadge: https://developers.google.com/profile/badges/playlists/solutions/barista/view exploreBadge: https://developers.google.com/profile/badges/playlists/solutions/barista/learn quizBadge: https://developers.google.com/profile/badges/playlists/solutions/barista/quiz buildBadge: https://developers.google.com/profile/badges/playlists/solutions/barista/action