Unlocking Multimodal Video Transcription with Gemini

Requirement:

• A Google Cloud mission
• The Vertex AI API should be enabled for this mission

Gen AI SDK setting variables:

Study extra about setting up a project and a development environment.

Requirement:

Gen AI SDK setting variables:

• GOOGLE_GENAI_USE_VERTEXAI="False"
• GOOGLE_API_KEY="<API_KEY>"

Study extra about getting a Gemini API key from Google AI Studio.

import os
import sys
from collections.abc import Callable

from google import genai

# Guide setup (depart unchanged if setup is environment-defined)

# @markdown **Which API: Vertex AI or Google AI Studio?**
GOOGLE_GENAI_USE_VERTEXAI = True  # @param {sort: "boolean"}

# @markdown **Choice A - Google Cloud mission [+location]**
GOOGLE_CLOUD_PROJECT = ""  # @param {sort: "string"}
GOOGLE_CLOUD_LOCATION = "world"  # @param {sort: "string"}

# @markdown **Choice B - Google AI Studio API key**
GOOGLE_API_KEY = ""  # @param {sort: "string"}


def check_environment() -> bool:
    check_colab_user_authentication()
    return check_manual_setup() or check_vertex_ai() or check_colab() or check_local()


def check_manual_setup() -> bool:
    return check_define_env_vars(
        GOOGLE_GENAI_USE_VERTEXAI,
        GOOGLE_CLOUD_PROJECT.strip(),  # May need been pasted with line return
        GOOGLE_CLOUD_LOCATION,
        GOOGLE_API_KEY,
    )


def check_vertex_ai() -> bool:
    # Workbench and Colab Enterprise
    match os.getenv("VERTEX_PRODUCT", ""):
        case "WORKBENCH_INSTANCE":
            go
        case "COLAB_ENTERPRISE":
            if not running_in_colab_env():
                return False
        case _:
            return False

    return check_define_env_vars(
        True,
        os.getenv("GOOGLE_CLOUD_PROJECT", ""),
        os.getenv("GOOGLE_CLOUD_REGION", ""),
        "",
    )


def check_colab() -> bool:
    if not running_in_colab_env():
        return False

    # Colab Enterprise was checked earlier than, so that is Colab solely
    from google.colab import auth as colab_auth  # sort: ignore

    colab_auth.authenticate_user()

    # Use Colab Secrets and techniques (🗝️ icon in left panel) to retailer the setting variables
    # Secrets and techniques are non-public, seen solely to you and the notebooks that you choose
    # - Vertex AI: Retailer your settings as secrets and techniques
    # - Google AI: Immediately import your Gemini API key from the UI
    vertexai, mission, location, api_key = get_vars(get_colab_secret)

    return check_define_env_vars(vertexai, mission, location, api_key)


def check_local() -> bool:
    vertexai, mission, location, api_key = get_vars(os.getenv)

    return check_define_env_vars(vertexai, mission, location, api_key)


def running_in_colab_env() -> bool:
    # Colab or Colab Enterprise
    return "google.colab" in sys.modules


def check_colab_user_authentication() -> None:
    if running_in_colab_env():
        from google.colab import auth as colab_auth  # sort: ignore

        colab_auth.authenticate_user()


def get_colab_secret(secret_name: str, default: str) -> str:
    from google.colab import userdata  # sort: ignore

    strive:
        return userdata.get(secret_name)
    besides Exception as e:
        return default


def get_vars(getenv: Callable[[str, str], str]) -> tuple[bool, str, str, str]:
    # Restrict getenv calls to the minimal (might set off UI affirmation for secret entry)
    vertexai_str = getenv("GOOGLE_GENAI_USE_VERTEXAI", "")
    if vertexai_str:
        vertexai = vertexai_str.decrease() in ["true", "1"]
    else:
        vertexai = bool(getenv("GOOGLE_CLOUD_PROJECT", ""))

    mission = getenv("GOOGLE_CLOUD_PROJECT", "") if vertexai else ""
    location = getenv("GOOGLE_CLOUD_LOCATION", "") if mission else ""
    api_key = getenv("GOOGLE_API_KEY", "") if not mission else ""

    return vertexai, mission, location, api_key


def check_define_env_vars(
    vertexai: bool,
    mission: str,
    location: str,
    api_key: str,
) -> bool:
    match (vertexai, bool(mission), bool(location), bool(api_key)):
        case (True, True, _, _):
            # Vertex AI - Google Cloud mission [+location]
            location = location or "world"
            define_env_vars(vertexai, mission, location, "")
        case (True, False, _, True):
            # Vertex AI - API key
            define_env_vars(vertexai, "", "", api_key)
        case (False, _, _, True):
            # Google AI Studio - API key
            define_env_vars(vertexai, "", "", api_key)
        case _:
            return False

    return True


def define_env_vars(vertexai: bool, mission: str, location: str, api_key: str) -> None:
    os.environ["GOOGLE_GENAI_USE_VERTEXAI"] = str(vertexai)
    os.environ["GOOGLE_CLOUD_PROJECT"] = mission
    os.environ["GOOGLE_CLOUD_LOCATION"] = location
    os.environ["GOOGLE_API_KEY"] = api_key


def check_configuration(consumer: genai.Shopper) -> None:
    service = "Vertex AI" if consumer.vertexai else "Google AI Studio"
    print(f"Utilizing the {service} API", finish="")

    if consumer._api_client.mission:
        print(f' with mission "{consumer._api_client.mission[:7]}…"', finish="")
        print(f' in location "{consumer._api_client.location}"')
    elif consumer._api_client.api_key:
        api_key = consumer._api_client.api_key
        print(f' with API key "{api_key[:5]}…{api_key[-5:]}"', finish="")
        print(f" (in case of error, be certain it was created for {service})")

import enum
from dataclasses import dataclass
from datetime import timedelta

import IPython.show
import tenacity
from google.genai.errors import ClientError
from google.genai.sorts import (
    FileData,
    FinishReason,
    GenerateContentConfig,
    GenerateContentResponse,
    Half,
    VideoMetadata,
)


class Mannequin(enum.Enum):
    # Usually Obtainable (GA)
    GEMINI_2_0_FLASH = "gemini-2.0-flash"
    GEMINI_2_5_FLASH = "gemini-2.5-flash"
    GEMINI_2_5_PRO = "gemini-2.5-pro"
    # Default mannequin
    DEFAULT = GEMINI_2_0_FLASH


# Default configuration for extra deterministic outputs
DEFAULT_CONFIG = GenerateContentConfig(
    temperature=0.0,
    top_p=0.0,
    seed=42,  # Arbitrary fastened worth
)

YOUTUBE_URL_PREFIX = "https://www.youtube.com/watch?v="
CLOUD_STORAGE_URI_PREFIX = "gs://"


def url_for_youtube_id(youtube_id: str) -> str:
    return f"{YOUTUBE_URL_PREFIX}{youtube_id}"


class Video(enum.Enum):
    go


class TestVideo(Video):
    # For testing functions, video period is statically specified within the enum title
    # Suffix (ISO 8601 based mostly): _PT[<h>H][<m>M][<s>S]

    # Google DeepMind | The Podcast | Season 3 Trailer | 59s
    GDM_PODCAST_TRAILER_PT59S = url_for_youtube_id("0pJn3g8dfwk")
    # Google Maps | Stroll within the footsteps of Jane Goodall | 2min 42s
    JANE_GOODALL_PT2M42S = "gs://cloud-samples-data/video/JaneGoodall.mp4"
    # Google DeepMind | AlphaFold | The making of a scientific breakthrough | 7min 54s
    GDM_ALPHAFOLD_PT7M54S = url_for_youtube_id("gg7WjuFs8F4")
    # Brut | French reportage | 8min 28s
    BRUT_FR_DOGS_WATER_LEAK_PT8M28S = url_for_youtube_id("U_yYkb-ureI")
    # Google DeepMind | The Podcast | AI for science | 54min 23s
    GDM_AI_FOR_SCIENCE_FRONTIER_PT54M23S = url_for_youtube_id("nQKmVhLIGcs")
    # Google I/O 2025 | Developer Keynote | 1h 10min 03s
    GOOGLE_IO_DEV_KEYNOTE_PT1H10M03S = url_for_youtube_id("GjvgtwSOCao")
    # Google Cloud | Subsequent 2025 | Opening Keynote | 1h 40min 03s
    GOOGLE_CLOUD_NEXT_PT1H40M03S = url_for_youtube_id("Md4Fs-Zc3tg")
    # Google I/O 2025 | Keynote | 1h 56min 35s
    GOOGLE_IO_KEYNOTE_PT1H56M35S = url_for_youtube_id("o8NiE3XMPrM")


class ShowAs(enum.Enum):
    DONT_SHOW = enum.auto()
    TEXT = enum.auto()
    MARKDOWN = enum.auto()


@dataclass
class VideoSegment:
    begin: timedelta
    finish: timedelta


def generate_content(
    immediate: str,
    video: Video | None = None,
    video_segment: VideoSegment | None = None,
    mannequin: Mannequin | None = None,
    config: GenerateContentConfig | None = None,
    show_as: ShowAs = ShowAs.TEXT,
) -> None:
    immediate = immediate.strip()
    mannequin = mannequin or Mannequin.DEFAULT
    config = config or DEFAULT_CONFIG

    model_id = mannequin.worth
    if video:
        if not (video_part := get_video_part(video, video_segment)):
            return
        contents = [video_part, prompt]
        caption = f"{video.title} / {model_id}"
    else:
        contents = immediate
        caption = f"{model_id}"
    print(f" {caption} ".heart(80, "-"))

    for try in get_retrier():
        with try:
            response = consumer.fashions.generate_content(
                mannequin=model_id,
                contents=contents,
                config=config,
            )
            display_response_info(response)
            display_response(response, show_as)


def get_video_part(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
) -> Half | None:
    video_uri: str = video.worth

    if not consumer.vertexai:
        video_uri = convert_to_https_url_if_cloud_storage_uri(video_uri)
        if not video_uri.startswith(YOUTUBE_URL_PREFIX):
            print("Google AI Studio API: Solely YouTube URLs are presently supported")
            return None

    file_data = FileData(file_uri=video_uri, mime_type="video/*")
    video_metadata = get_video_part_metadata(video_segment, fps)

    return Half(file_data=file_data, video_metadata=video_metadata)


def get_video_part_metadata(
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
) -> VideoMetadata:
    def offset_as_str(offset: timedelta) -> str:
        return f"{offset.total_seconds()}s"

    if video_segment:
        start_offset = offset_as_str(video_segment.begin)
        end_offset = offset_as_str(video_segment.finish)
    else:
        start_offset = None
        end_offset = None

    return VideoMetadata(start_offset=start_offset, end_offset=end_offset, fps=fps)


def convert_to_https_url_if_cloud_storage_uri(uri: str) -> str:
    if uri.startswith(CLOUD_STORAGE_URI_PREFIX):
        return f"https://storage.googleapis.com/{uri.removeprefix(CLOUD_STORAGE_URI_PREFIX)}"
    else:
        return uri


def get_retrier() -> tenacity.Retrying:
    return tenacity.Retrying(
        cease=tenacity.stop_after_attempt(7),
        wait=tenacity.wait_incrementing(begin=10, increment=1),
        retry=should_retry_request,
        reraise=True,
    )


def should_retry_request(retry_state: tenacity.RetryCallState) -> bool:
    if not retry_state.end result:
        return False
    err = retry_state.end result.exception()
    if not isinstance(err, ClientError):
        return False
    print(f"❌ ClientError {err.code}: {err.message}")

    retry = False
    match err.code:
        case 400 if err.message will not be None and " strive once more " in err.message:
            # Workshop: mission accessing Cloud Storage for the primary time (service agent provisioning)
            retry = True
        case 429:
            # Workshop: momentary mission with 1 QPM quota
            retry = True
    print(f"🔄 Retry: {retry}")

    return retry


def display_response_info(response: GenerateContentResponse) -> None:
    if usage_metadata := response.usage_metadata:
        if usage_metadata.prompt_token_count:
            print(f"Enter tokens   : {usage_metadata.prompt_token_count:9,d}")
        if usage_metadata.candidates_token_count:
            print(f"Output tokens  : {usage_metadata.candidates_token_count:9,d}")
        if usage_metadata.thoughts_token_count:
            print(f"Ideas tokens: {usage_metadata.thoughts_token_count:9,d}")
    if not response.candidates:
        print("❌ No `response.candidates`")
        return
    if (finish_reason := response.candidates[0].finish_reason) != FinishReason.STOP:
        print(f"❌ {finish_reason = }")
    if not response.textual content:
        print("❌ No `response.textual content`")
        return


def display_response(
    response: GenerateContentResponse,
    show_as: ShowAs,
) -> None:
    if show_as == ShowAs.DONT_SHOW:
        return
    if not (response_text := response.textual content):
        return
    response_text = response.textual content.strip()

    print(" begin of response ".heart(80, "-"))
    match show_as:
        case ShowAs.TEXT:
            print(response_text)
        case ShowAs.MARKDOWN:
            display_markdown(response_text)
    print(" finish of response ".heart(80, "-"))


def display_markdown(markdown: str) -> None:
    IPython.show.show(IPython.show.Markdown(markdown))


def display_video(video: Video) -> None:
    video_url = convert_to_https_url_if_cloud_storage_uri(video.worth)
    assert video_url.startswith("https://")

    video_width = 600
    if video_url.startswith(YOUTUBE_URL_PREFIX):
        youtube_id = video_url.removeprefix(YOUTUBE_URL_PREFIX)
        ipython_video = IPython.show.YouTubeVideo(youtube_id, width=video_width)
    else:
        ipython_video = IPython.show.Video(video_url, width=video_width)

    display_markdown(f"### Video ([source]({video_url}))")
    IPython.show.show(ipython_video)

------------------- GDM_ALPHAFOLD_PT7M54S / gemini-2.0-flash -------------------
Enter tokens   :   134,177
Output tokens  :     2,689
------------------------------ begin of response -------------------------------
[00:02] ? - We have found extra concerning the world than every other civilization earlier than us.
[00:08] ? - However now we have been caught on this one downside.
[00:11] ? - How do proteins fold up?
[00:13] ? - How do proteins go from a string of amino acids to a compact form that acts as a machine and drives life?
[00:22] ? - Once you discover out about proteins, it is extremely thrilling.
[00:25] ? - You can consider them as little organic nano machines.
[00:28] ? - They're basically the basic constructing blocks that energy every part dwelling on this planet.
[00:34] ? - If we will reliably predict protein constructions utilizing AI, that would change the way in which we perceive the pure world.
[00:46] ? - Protein folding is one in all these holy grail sort issues in biology.
[00:50] Demis Hassabis - We have all the time hypothesized that AI must be useful to make these varieties of massive scientific breakthroughs extra rapidly.
[00:58] ? - After which I will in all probability be  little tunings which may make a distinction.
[01:02] ? - It must be making a histogram on and a background ability.
[01:04] ? - We have been engaged on our system AlphaFold actually onerous now for over two years.
[01:08] ? - Quite than having to do painstaking experiments, sooner or later biologists may be capable of as an alternative depend on AI strategies to immediately predict constructions rapidly and effectively.
[01:17] Kathryn Tunyasuvunakool - Usually talking, biologists are usually fairly skeptical of computational work, and I feel that skepticism is wholesome and I respect it, however I really feel very enthusiastic about what AlphaFold can obtain.
[01:28] Andrew Senior - CASP is once we, we are saying, look, DeepMind is doing protein folding.
[01:31] Andrew Senior - That is how good we're, and possibly it is higher than everyone else, possibly it is not.
[01:37] ? - We determined to enter CASP competitors as a result of it represented the Olympics of protein folding.
[01:44] John Moult - CASP, we began to try to velocity up the answer to the protein folding downside.
[01:50] John Moult - Once we began CASP in 1994, I actually was naive about how onerous this was going to be.
[01:58] ? - It was very cumbersome to try this as a result of it took a very long time.
[02:01] ? - Let's examine what, what, what are we doing nonetheless to enhance?
[02:03] ? - Sometimes 100 completely different teams from around the globe take part in CASP, and we take a set of 100 proteins and we ask the teams to ship us what they suppose the constructions appear like.
[02:15] ? - We will attain 57.9 GDT on CASP 12 floor fact.
[02:19] John Jumper - CASP has a metric on which you may be scored, which is that this GDT metric.
[02:25] John Jumper - On a scale of zero to 100, you'll count on a GDT over 90 to be an answer to the issue.
[02:33] ? - If we do obtain this, this has unbelievable medical relevance.
[02:37] Pushmeet Kohli - The implications are immense, from how illnesses progress, how one can uncover new medication.
[02:45] Pushmeet Kohli - It is limitless.
[02:46] ? - I needed to make a, a extremely easy system and the outcomes have been surprisingly good.
[02:50] ? - The crew acquired some outcomes with a brand new method, not solely is it extra correct, but it surely's a lot quicker than the outdated system.
[02:56] ? - I feel we'll considerably exceed what we're doing proper now.
[02:59] ? - This can be a sport, sport changer, I feel.
[03:01] John Moult - In CASP 13, one thing very vital had occurred.
[03:06] John Moult - For the primary time, we noticed the efficient software of synthetic intelligence.
[03:11] ? - We have superior the cutting-edge within the subject, in order that's incredible, however we nonetheless acquired a protracted option to go earlier than we have solved it.
[03:18] ? - The shapes had been now roughly appropriate for most of the proteins, however the particulars, precisely the place every atom sits, which is basically what we'd name an answer, we're not but there.
[03:29] ? - It does not assist when you have the tallest ladder when you are going to the moon.
[03:33] ? - We hit slightly little bit of a brick wall, um, since we received CASP, then it was again to the drafting board and like what are our new concepts?
[03:41] ? - Um, after which it is taken a short while, I might say, for them to get again to the place they had been, however with the brand new concepts.
[03:51] ? - They will go additional, proper?
[03:52] ? - So, um, in order that's a extremely vital second.
[03:55] ? - I've seen that second so many occasions now, however I do know what meaning now, and I do know that is the time now to press.
[04:02] ? - We have to double down and go as quick as potential from right here.
[04:05] ? - I feel we have no time to lose.
[04:07] ? - So the intention is to enter CASP once more.
[04:09] ? - CASP is deeply anxious.
[04:12] ? - There's one thing bizarre occurring with, um, the training as a result of it's studying one thing that is correlated with GDT, but it surely's not calibrated.
[04:18] ? - I really feel barely uncomfortable.
[04:20] ? - We must be studying this, you understand, within the blink of an eye fixed.
[04:23] ? - The know-how advancing outdoors DeepMind can also be doing unbelievable work.
[04:27] Richard Evans - And there is all the time the likelihood one other crew has come someplace on the market subject that we do not even learn about.
[04:32] ? - Somebody requested me, properly, ought to we panic now?
[04:33] ? - After all, we must always have been panicking earlier than.
[04:35] ? - It does appear to do higher, however nonetheless does not do fairly in addition to the perfect mannequin.
[04:39] ? - Um, so it appears to be like like there's room for enchancment.
[04:42] ? - There's all the time a threat that you've got missed one thing, and that is why blind assessments like CASP are so vital to validate whether or not our outcomes are actual.
[04:49] ? - Clearly, I am excited to see how CASP 14 goes.
[04:51] ? - My expectation is we get our heads down, we deal with the complete objective, which is to unravel the entire downside.
[05:14] ? - We had been ready for CASP to begin on April fifteenth as a result of that is when it was initially scheduled to begin, and it has been delayed by a month on account of coronavirus.
[05:24] ? - I actually miss everybody.
[05:25] ? - No, I struggled slightly bit simply sort of getting right into a routine, particularly, uh, my spouse, she got here down with the, the virus.
[05:32] ? - I imply, fortunately it did not end up too critical.
[05:34] ? - CASP began on Monday.
[05:37] Demis Hassabis - Can I simply verify this diagram you have acquired right here, John, this one the place we ask floor fact.
[05:40] Demis Hassabis - Is that this one we have finished badly on?
[05:42] ? - We're really fairly good on this area.
[05:43] ? - If you happen to think about that we hadn't have stated it got here round this manner, however had put it in.
[05:47] ? - Yeah, and that as an alternative.
[05:48] ? - Yeah.
[05:49] ? - One of many hardest proteins we have gotten in CASP so far is a SARS-CoV-2 protein, uh, known as Orf8.
[05:55] ? - Orf8 is a coronavirus protein.
[05:57] ? - We tried actually onerous to enhance our prediction, like actually, actually onerous, in all probability probably the most time that now we have ever spent on a single goal.
[06:05] ? - So we're about two-thirds of the way in which by CASP, and we have gotten three solutions again.
[06:11] ? - We now have a floor fact for Orf8, which is among the coronavirus proteins.
[06:17] ? - And it seems we did very well in predicting that.
[06:20] Demis Hassabis - Wonderful job, everybody, the entire crew.
[06:23] Demis Hassabis - It has been an unbelievable effort.
[06:24] John Moult - Right here what we noticed in CASP 14 was a gaggle delivering atomic accuracy off the bat, basically fixing what in our world is 2 issues.
[06:34] John Moult - How do you look to search out the suitable answer, after which how do you acknowledge you have acquired the suitable answer while you're there?
[06:41] ? - All proper, are we, are we principally right here?
[06:46] ? - I'll learn an e-mail.
[06:48] ? - Uh, I acquired this from John Moult.
[06:50] ? - Now I will simply learn it.
[06:51] ? - It says, John, as I count on you understand, your group has carried out amazingly properly in CASP 14, each relative to different teams and in absolute mannequin accuracy.
[07:02] ? - Congratulations on this work.
[07:05] ? - It's actually excellent.
[07:07] Demis Hassabis - AlphaFold represents an enormous leap ahead that I hope will actually speed up drug discovery and assist us to higher perceive illness.
[07:13] John Moult - It is fairly mind-blowing.
[07:16] John Moult - You recognize, these outcomes had been, for me, having labored on this downside so lengthy, after many, many stops and begins and can this ever get there, immediately this can be a answer.
[07:28] John Moult - We have solved the issue.
[07:29] John Moult - This offers you such pleasure about the way in which science works, about how one can by no means see precisely and even roughly what is going on to occur subsequent.
[07:37] John Moult - There are all the time these surprises, and that basically, as a scientist, is what retains you going.
[07:41] John Moult - What is going on to be the subsequent shock?
------------------------------- finish of response --------------------------------

------------------------------- gemini-2.0-flash -------------------------------
Enter tokens   :        18
Output tokens  :     1,814
------------------------------ begin of response -------------------------------

## Terminology Used for Video Transcriptions

Here is a breakdown of widespread terminology used when discussing video transcriptions:

*   **Transcription:** The final time period for changing audio or video content material into textual content.
*   **Video Transcription:** Particularly refers to transcribing the audio portion of a video.
*   **Verbatim Transcription:** A transcription that features each spoken phrase, together with filler phrases ("um," "ah"), false begins, stutters, and background noises (if related).  It goals for a extremely correct illustration of the spoken phrase.
*   **Clear Verbatim Transcription (or Edited Transcription):** A transcription that removes filler phrases, false begins, and stutters to create a extra readable and polished textual content. It focuses on conveying the which means clearly.
*   **Clever Verbatim Transcription:** A center floor between verbatim and clear verbatim. It removes some filler phrases and stutters however retains components that contribute to the speaker's fashion or the context of the dialog.
*   **Abstract Transcription:** A quick overview of the video's content material, highlighting the details.
*   **Timecoding (or Timestamping):** Including timestamps to the transcription to point when every part of textual content was spoken within the video.  That is essential for accessibility and navigation.
*   **Speaker Identification:** Figuring out and labeling completely different audio system within the video.
*   **Captioning (or Subtitling):** Creating textual content variations of the video's audio which can be displayed on the display, normally synchronized with the video.  Captions are sometimes used for accessibility (for deaf or hard-of-hearing viewers) and for viewers preferring to learn alongside.
*   **Closed Captions:** Captions that may be turned on or off by the viewer.
*   **Open Captions:** Captions which can be completely embedded within the video and can't be turned off.
*   **SRT (SubRip Subtitle):** A standard file format for storing captions and subtitles.
*   **VTT (Video Textual content Tracks):** A extra fashionable file format for captions and subtitles, typically used with HTML5 video.
*   **Burned-in Captions:** Captions which can be completely embedded into the video file.
*   **Transcriber:** The particular person or software program that performs the transcription.
*   **Accuracy Charge:** The proportion of phrases transcribed accurately.
*   **Turnaround Time:** The time it takes to finish the transcription.

## Typical Output Instance

As an instance now we have a brief video clip of an interview. Here is how various kinds of transcriptions may look:

**Situation:** A brief clip of an interview with a chef discussing their new restaurant.

**1. Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is, uh, nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Properly, um, it is, it has been a dream of mine for, like, years, you understand?  To, like, open a spot that basically focuses on, um, seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can individuals count on?
00:00:25 --> 00:00:32 Chef: We're doing, uh, fashionable American, however with, like, an actual emphasis on, uh, native farms and, you understand, sustainable practices.
```

**2. Clear Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Properly, it has been a dream of mine for years to open a spot that basically focuses on seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can individuals count on?
00:00:25 --> 00:00:32 Chef: We're doing fashionable American, however with an actual emphasis on native farms and sustainable practices.
```

**3. Clever Verbatim Transcription (with Timecoding):**

```
00:00:00 --> 00:00:03 Interviewer: So, Chef, welcome to the present!
00:00:03 --> 00:00:06 Chef: Thanks for having me. It is nice to be right here.
00:00:06 --> 00:00:12 Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."
00:00:12 --> 00:00:20 Chef: Properly, it has been a dream of mine for, like, years, you understand? To open a spot that basically focuses on seasonal elements.
00:00:20 --> 00:00:25 Interviewer: And what sort of delicacies can individuals count on?
00:00:25 --> 00:00:32 Chef: We're doing fashionable American, however with, like, an actual emphasis on native farms and, you understand, sustainable practices.
```

**4. Abstract Transcription:**

```
Chef discusses their new restaurant, "The Golden Spoon," which focuses on seasonal elements and fashionable American delicacies with an emphasis on native farms and sustainable practices.
```

**5. SRT File (for Captions):**

```
1
00:00:00,000 --> 00:00:03,000
Interviewer: So, Chef, welcome to the present!

2
00:00:03,000 --> 00:00:06,000
Chef: Thanks for having me. It is nice to be right here.

3
00:00:06,000 --> 00:00:12,000
Interviewer: So, inform us slightly bit about your new restaurant, "The Golden Spoon."

4
00:00:12,000 --> 00:00:20,000
Chef: Properly, it has been a dream of mine for years to open a spot that basically focuses on seasonal elements.

5
00:00:20,000 --> 00:00:25,000
Interviewer: And what sort of delicacies can individuals count on?

6
00:00:25,000 --> 00:00:32,000
Chef: We're doing fashionable American, however with an actual emphasis on native farms and sustainable practices.
```

**Key Variations:**

*   **Verbatim:** Captures each element, together with imperfections in speech.
*   **Clear Verbatim:**  Focuses on readability and readability by eradicating pointless components.
*   **Clever Verbatim:** A stability between accuracy and readability.
*   **Abstract:** Gives a concise overview.
*   **SRT:**  A structured file format for displaying captions synchronized with the video.

The perfect sort of transcription depends upon the particular wants of the mission.  For authorized or analysis functions, a verbatim transcription could be crucial. For normal accessibility or advertising functions, a clear verbatim or clever verbatim transcription is commonly most well-liked.  Abstract transcriptions are helpful for rapidly understanding the content material of a video.  SRT recordsdata are important for including captions to movies.

------------------------------- finish of response --------------------------------

----------------- GDM_PODCAST_TRAILER_PT59S / gemini-2.0-flash -----------------
Enter tokens   :    16,800
Output tokens  :       635
------------------------------ begin of response -------------------------------

[
  {
    "start": "00:00",
    "text": "Do I have to call you Sir Demis now?",
    "voice": 1
  },
  {
    "start": "00:01",
    "text": "Oh, you don't. Absolutely not.",
    "voice": 2
  },
  {
    "start": "00:03",
    "text": "Welcome to Google Deep Mind the podcast with me, your host Professor Hannah Fry.",
    "voice": 1
  },
  {
    "start": "00:06",
    "text": "We want to take you to the heart of where these ideas are coming from. We want to introduce you to the people who are leading the design of our collective future.",
    "voice": 1
  },
  {
    "start": "00:19",
    "text": "Getting the safety right is probably, I'd say, one of the most important challenges of our time. I want safe and capable.",
    "voice": 3
  },
  {
    "start": "00:26",
    "text": "I want a bridge that will not collapse.",
    "voice": 3
  },
  {
    "start": "00:30",
    "text": "just give these scientists a superpower that they had not imagined earlier.",
    "voice": 4
  },
  {
    "start": "00:34",
    "text": "autonomous vehicles. It's hard to fathom that when you're working on a search engine.",
    "voice": 5
  },
  {
    "start": "00:38",
    "text": "We may see entirely new genre or entirely new forms of art come up. There may be a new word that is not music, painting, photography, movie making, and that AI will have helped us create it.",
    "voice": 6
  },
  {
    "start": "00:48",
    "text": "You really want AGI to be able to peer into the mysteries of the universe.",
    "voice": 1
  },
  {
    "start": "00:51",
    "text": "Yes, quantum mechanics, string theory, well, and the nature of reality.",
    "voice": 2
  },
  {
    "start": "00:55",
    "text": "Ow.",
    "voice": 1
  },
  {
    "start": "00:56",
    "text": "the magic of AI.",
    "voice": 6
  }
]

------------------------------- finish of response --------------------------------

----------------- GDM_PODCAST_TRAILER_PT59S / gemini-2.0-flash -----------------
Enter tokens   :    16,920
Output tokens  :       806
------------------------------ begin of response -------------------------------

{
  "task1_transcripts": [
    {
      "start": "00:00",
      "text": "Do I have to call you Sir Demis now?",
      "voice": 1
    },
    {
      "start": "00:01",
      "text": "Oh, you don't. Absolutely not.",
      "voice": 2
    },
    {
      "start": "00:04",
      "text": "Welcome to Google Deep Mind the podcast with me, your host Professor Hannah Fry.",
      "voice": 1
    },
    {
      "start": "00:06",
      "text": "We want to take you to the heart of where these ideas are coming from. We want to introduce you to the people who are leading the design of our collective future.",
      "voice": 1
    },
    {
      "start": "00:19",
      "text": "Getting the safety right is probably, I'd say, one of the most important challenges of our time. I want safe and capable.",
      "voice": 3
    },
    {
      "start": "00:26",
      "text": "I want a bridge that will not collapse.",
      "voice": 3
    },
    {
      "start": "00:30",
      "text": "That just give these scientists a superpower that they had not imagined earlier.",
      "voice": 4
    },
    {
      "start": "00:34",
      "text": "autonomous vehicles. It's hard to fathom that when you're working on a search engine.",
      "voice": 5
    },
    {
      "start": "00:38",
      "text": "We may see entirely new genre or entirely new forms of art come up. There may be a new word that is not music, painting, photography, movie making, and that AI will have helped us create it.",
      "voice": 6
    },
    {
      "start": "00:48",
      "text": "You really want AGI to be able to peer into the mysteries of the universe.",
      "voice": 1
    },
    {
      "start": "00:51",
      "text": "Yes, quantum mechanics, string theory, well, and the nature of reality.",
      "voice": 2
    },
    {
      "start": "00:55",
      "text": "Ow.",
      "voice": 1
    },
    {
      "start": "00:56",
      "text": "the magic of AI.",
      "voice": 6
    }
  ],
  "task2_speakers": [
    {
      "voice": 1,
      "name": "Professor Hannah Fry"
    },
    {
      "voice": 2,
      "name": "Demis Hassabis"
    },
    {
      "voice": 3,
      "name": "Anca Dragan"
    },
    {
      "voice": 4,
      "name": "Pushmeet Kohli"
    },
    {
      "voice": 5,
      "name": "Jeff Dean"
    },
    {
      "voice": 6,
      "name": "Douglas Eck"
    }
  ]
}

------------------------------- finish of response --------------------------------

import re

import pydantic
from google.genai.sorts import MediaResolution, ThinkingConfig

SamplingFrameRate = float

VIDEO_TRANSCRIPTION_PROMPT = """
**Process 1 - Transcripts**

- Watch the video and hear fastidiously to the audio.
- Determine every distinctive voice utilizing a `voice` ID (1, 2, 3, and so forth.).
- Transcribe the video's audio verbatim with voice diarization.
- Embody the `begin` timecode ({timecode_spec}) for every speech phase.

**Process 2 - Audio system**

- For every `voice` ID from Process 1, extract details about the corresponding speaker.
- Use visible and audio cues.
- If a chunk of knowledge can't be discovered, use a query mark (`?`) as the worth.
"""
NOT_FOUND = "?"


class Transcript(pydantic.BaseModel):
    begin: str
    textual content: str
    voice: int


class Speaker(pydantic.BaseModel):
    voice: int
    title: str
    firm: str
    place: str
    role_in_video: str


class VideoTranscription(pydantic.BaseModel):
    task1_transcripts: listing[Transcript] = pydantic.Area(default_factory=listing)
    task2_speakers: listing[Speaker] = pydantic.Area(default_factory=listing)


def get_generate_content_config(mannequin: Mannequin, video: Video) -> GenerateContentConfig:
    media_resolution = get_media_resolution_for_video(video)
    thinking_config = get_thinking_config(mannequin)

    return GenerateContentConfig(
        temperature=DEFAULT_CONFIG.temperature,
        top_p=DEFAULT_CONFIG.top_p,
        seed=DEFAULT_CONFIG.seed,
        response_mime_type="software/json",
        response_schema=VideoTranscription,
        media_resolution=media_resolution,
        thinking_config=thinking_config,
    )


def get_video_duration(video: Video) -> timedelta | None:
    # For testing functions, video period is statically specified within the enum title
    # Suffix (ISO 8601 based mostly): _PT[<h>H][<m>M][<s>S]
    # For manufacturing,
    # - fetch durations dynamically or retailer them individually
    # - bear in mind video VideoMetadata.start_offset & VideoMetadata.end_offset
    regex = r"_PT(?:(d+)H)?(?:(d+)M)?(?:(d+)S)?$"
    if not (match := re.search(regex, video.title)):
        print(f"⚠️ No period data in {video.title}. Will use defaults.")
        return None

    h_str, m_str, s_str = match.teams()
    return timedelta(
        hours=int(h_str) if h_str will not be None else 0,
        minutes=int(m_str) if m_str will not be None else 0,
        seconds=int(s_str) if s_str will not be None else 0,
    )


def get_media_resolution_for_video(video: Video) -> MediaResolution | None:
    if not (video_duration := get_video_duration(video)):
        return None  # Default

    # For testing functions, that is based mostly on video period, as our brief movies are usually extra detailed
    less_than_five_minutes = video_duration < timedelta(minutes=5)
    if less_than_five_minutes:
        media_resolution = MediaResolution.MEDIA_RESOLUTION_MEDIUM
    else:
        media_resolution = MediaResolution.MEDIA_RESOLUTION_LOW

    return media_resolution


def get_sampling_frame_rate_for_video(video: Video) -> SamplingFrameRate | None:
    sampling_frame_rate = None  # Default (1 FPS for present fashions)

    # [Optional] Outline a customized FPS: 0.0 < sampling_frame_rate <= 24.0

    return sampling_frame_rate


def get_timecode_spec_for_model_and_video(mannequin: Mannequin, video: Video) -> str:
    timecode_spec = "MM:SS"  # Default

    match mannequin:
        case Mannequin.GEMINI_2_0_FLASH:  # Helps MM:SS
            go
        case Mannequin.GEMINI_2_5_FLASH | Mannequin.GEMINI_2_5_PRO:  # Assist MM:SS and H:MM:SS
            period = get_video_duration(video)
            one_hour_or_more = period will not be None and timedelta(hours=1) <= period
            if one_hour_or_more:
                timecode_spec = "MM:SS or H:MM:SS"
        case _:
            assert False, "Add timecode format for brand spanking new mannequin"

    return timecode_spec


def get_thinking_config(mannequin: Mannequin) -> ThinkingConfig | None:
    # Examples of considering configurations (Gemini 2.5 fashions)
    match mannequin:
        case Mannequin.GEMINI_2_5_FLASH:  # Pondering disabled
            return ThinkingConfig(thinking_budget=0, include_thoughts=False)
        case Mannequin.GEMINI_2_5_PRO:  # Minimal considering funds and no summarized ideas
            return ThinkingConfig(thinking_budget=128, include_thoughts=False)
        case _:
            return None  # Default


def get_video_transcription_from_response(
    response: GenerateContentResponse,
) -> VideoTranscription:
    if not isinstance(response.parsed, VideoTranscription):
        print("❌ Couldn't parse the JSON response")
        return VideoTranscription()  # Empty transcription

    return response.parsed


def get_video_transcription(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
    immediate: str | None = None,
    mannequin: Mannequin | None = None,
) -> VideoTranscription:
    mannequin = mannequin or Mannequin.DEFAULT
    model_id = mannequin.worth

    fps = fps or get_sampling_frame_rate_for_video(video)
    video_part = get_video_part(video, video_segment, fps)
    if not video_part:  # Unsupported supply, return an empty transcription
        return VideoTranscription()
    if immediate is None:
        timecode_spec = get_timecode_spec_for_model_and_video(mannequin, video)
        immediate = VIDEO_TRANSCRIPTION_PROMPT.format(timecode_spec=timecode_spec)
    contents = [video_part, prompt.strip()]

    config = get_generate_content_config(mannequin, video)

    print(f" {video.title} / {model_id} ".heart(80, "-"))
    response = None
    for try in get_retrier():
        with try:
            response = consumer.fashions.generate_content(
                mannequin=model_id,
                contents=contents,
                config=config,
            )
            display_response_info(response)

    assert isinstance(response, GenerateContentResponse)
    return get_video_transcription_from_response(response)

import itertools
from collections.abc import Callable, Iterator

from pandas import DataFrame, Collection
from pandas.io.codecs.fashion import Styler
from pandas.io.codecs.style_render import CSSDict

BGCOLOR_COLUMN = "bg_color"  # Hidden column to retailer row background colours


def yield_known_speaker_color() -> Iterator[str]:
    PAL_40 = ("#669DF6", "#EE675C", "#FCC934", "#5BB974")
    PAL_30 = ("#8AB4F8", "#F28B82", "#FDD663", "#81C995")
    PAL_20 = ("#AECBFA", "#F6AEA9", "#FDE293", "#A8DAB5")
    PAL_10 = ("#D2E3FC", "#FAD2CF", "#FEEFC3", "#CEEAD6")
    PAL_05 = ("#E8F0FE", "#FCE8E6", "#FEF7E0", "#E6F4EA")
    return itertools.cycle([*PAL_40, *PAL_30, *PAL_20, *PAL_10, *PAL_05])


def yield_unknown_speaker_color() -> Iterator[str]:
    GRAYS = ["#80868B", "#9AA0A6", "#BDC1C6", "#DADCE0", "#E8EAED", "#F1F3F4"]
    return itertools.cycle(GRAYS)


def get_color_for_voice_mapping(audio system: listing[Speaker]) -> dict[int, str]:
    known_speaker_color = yield_known_speaker_color()
    unknown_speaker_color = yield_unknown_speaker_color()

    mapping: dict[int, str] = {}
    for speaker in audio system:
        if speaker.title != NOT_FOUND:
            coloration = subsequent(known_speaker_color)
        else:
            coloration = subsequent(unknown_speaker_color)
        mapping[speaker.voice] = coloration

    return mapping


def get_table_styler(df: DataFrame) -> Styler:
    def join_styles(types: listing[str]) -> str:
        return ";".be a part of(types)

    table_css = [
        "color: #202124",
        "background-color: #BDC1C6",
        "border: 0",
        "border-radius: 0.5rem",
        "border-spacing: 0px",
        "outline: 0.5rem solid #BDC1C6",
        "margin: 1rem 0.5rem",
    ]
    th_css = ["background-color: #E8EAED"]
    th_td_css = ["text-align:left", "padding: 0.25rem 1rem"]
    table_styles = [
        CSSDict(selector="", props=join_styles(table_css)),
        CSSDict(selector="th", props=join_styles(th_css)),
        CSSDict(selector="th,td", props=join_styles(th_td_css)),
    ]

    return df.fashion.set_table_styles(table_styles).conceal()


def change_row_bgcolor(row: Collection) -> listing[str]:
    fashion = f"background-color:{row[BGCOLOR_COLUMN]}"
    return [style] * len(row)


def display_table(yield_rows: Callable[[], Iterator[list[str]]]) -> None:
    knowledge = yield_rows()
    df = DataFrame(columns=subsequent(knowledge), knowledge=knowledge)
    styler = get_table_styler(df)
    styler.apply(change_row_bgcolor, axis=1)
    styler.conceal([BGCOLOR_COLUMN], axis="columns")

    html = styler.to_html()
    IPython.show.show(IPython.show.HTML(html))


def display_speakers(transcription: VideoTranscription) -> None:
    def sanitize_field(s: str, symbol_if_unknown: str) -> str:
        return symbol_if_unknown if s == NOT_FOUND else s

    def yield_rows() -> Iterator[list[str]]:
        yield ["voice", "name", "company", "position", "role_in_video", BGCOLOR_COLUMN]

        color_for_voice = get_color_for_voice_mapping(transcription.task2_speakers)
        for speaker in transcription.task2_speakers:
            yield [
                str(speaker.voice),
                sanitize_field(speaker.name, NOT_FOUND),
                sanitize_field(speaker.company, NOT_FOUND),
                sanitize_field(speaker.position, NOT_FOUND),
                sanitize_field(speaker.role_in_video, NOT_FOUND),
                color_for_voice.get(speaker.voice, "red"),
            ]

    display_markdown(f"### Audio system ({len(transcription.task2_speakers)})")
    display_table(yield_rows)


def display_transcripts(transcription: VideoTranscription) -> None:
    def yield_rows() -> Iterator[list[str]]:
        yield ["start", "speaker", "transcript", BGCOLOR_COLUMN]

        color_for_voice = get_color_for_voice_mapping(transcription.task2_speakers)
        speaker_for_voice = {
            speaker.voice: speaker for speaker in transcription.task2_speakers
        }
        previous_voice = None
        for transcript in transcription.task1_transcripts:
            current_voice = transcript.voice
            speaker_label = ""
            if speaker := speaker_for_voice.get(current_voice, None):
                if speaker.title != NOT_FOUND:
                    speaker_label = speaker.title
                elif speaker.place != NOT_FOUND:
                    speaker_label = f"[voice {current_voice}][{speaker.position}]"
                elif speaker.role_in_video != NOT_FOUND:
                    speaker_label = f"[voice {current_voice}][{speaker.role_in_video}]"
            if not speaker_label:
                speaker_label = f"[voice {current_voice}]"
            yield [
                transcript.start,
                speaker_label if current_voice != previous_voice else '"',
                transcript.text,
                color_for_voice.get(current_voice, "red"),
            ]
            previous_voice = current_voice

    display_markdown(f"### Transcripts ({len(transcription.task1_transcripts)})")
    display_table(yield_rows)


def transcribe_video(
    video: Video,
    video_segment: VideoSegment | None = None,
    fps: float | None = None,
    immediate: str | None = None,
    mannequin: Mannequin | None = None,
) -> None:
    display_video(video)
    transcription = get_video_transcription(video, video_segment, fps, immediate, mannequin)
    display_speakers(transcription)
    display_transcripts(transcription)