media_analyzer

21class MediaAnalyzer:
22    """Analyze media using a machine learning models, file based analysis, and exif data."""
23
24    config: FullAnalyzerConfig
25
26    def __init__(self, config: AnalyzerSettings | None = None) -> None:
27        """Initialize the media analyzer with the given configuration."""
28        if config is None:
29            config = AnalyzerSettings()
30        embedder = get_embedder_by_provider(config.embedder_provider)
31        self.config = FullAnalyzerConfig(
32            llm=get_llm_by_provider(config.llm_provider),
33            captioner=get_captioner_by_provider(config.captions_provider),
34            ocr=ResnetTesseractOCR(),
35            object_detector=ResnetObjectDetection(),
36            facial_recognition=InsightFacialRecognition(),
37            embedder=embedder,
38            settings=config,
39        )
40
41    def analyze(self, input_media: InputMedia) -> MediaAnalyzerOutput:
42        """Analyze the given photo or video."""
43        image_data, frame_data = run_metadata_pipeline(input_media, self.config)
44        image_data_output = ImageDataOutput(
45            path=image_data.path,
46            exif=image_data.exif,
47            data_url=image_data.data_url,
48            gps=image_data.gps,
49            time=image_data.time,
50            weather=image_data.weather,
51            tags=image_data.tags,
52        )
53        frame_output = [
54            FrameDataOutput(
55                ocr=frame.ocr,
56                embedding=frame.embedding,
57                faces=frame.faces,
58                summary=frame.summary,
59                caption_data=frame.caption_data,
60                objects=frame.objects,
61                measured_quality=frame.measured_quality,
62                color=frame.color,
63            )
64            for frame in frame_data
65        ]
66        return MediaAnalyzerOutput(image_data=image_data_output, frame_data=frame_output)
67
68    def photo(self, image_path: Path) -> MediaAnalyzerOutput:
69        """Analyze a photo."""
70        return self.analyze(InputMedia(image_path, frames=[image_path]))

23@dataclass
24class MediaAnalyzerOutput:
25    """Output of the media-analyzer package.
26
27    Attributes:
28        image_data: File based analysis.
29        frame_data: Visual analysis for the frames given in the input.
30    """
31
32    image_data: ImageDataOutput
33    frame_data: list[FrameDataOutput]

 9@dataclass
10class InputMedia:
11    """Input for the media-analyzer package.
12
13    Attributes:
14        path: The path to the photo or video file.
15        frames: A list of frame paths. In case of a photo, one frame is supplied,
16            for a video you can generate multiple frames and submit them for analysis.
17    """
18
19    path: Path
20    frames: list[Path]

20@dataclass
21class AnalyzerSettings:
22    """Configuration settings for the media analysis pipeline.
23
24    This class contains various options for configuring how photo and video files
25    are analyzed, including language settings for OCR, the selection of providers
26    for captions and LLMs, and thresholds for different detection modules.
27
28    Attributes:
29        media_languages: The languages used for OCR.
30        theme_color_variant: The color variant used for the generated theme.
31        theme_contrast_level: The contrast level used for the generated theme.
32        captions_provider: The provider to be used for generating captions.
33        llm_provider: The provider for the large language model (LLM),
34            which can be used for summaries and captions.
35        enable_text_summary: Flag to enable or disable image summarization, uses LLM so is slow.
36        enable_document_summary: Flag to enable or disable document summaries, uses LLM so is slow.
37        document_detection_threshold: Threshold for detecting documents in images [0-100].
38        face_detection_threshold: Threshold for face detection [0-1].
39        enabled_file_modules: The set of modules used for file-based analysis.
40        enabled_visual_modules: The set of modules for visual analysis.
41    """
42
43    media_languages: tuple[str, ...] = ("nld", "eng")
44    theme_contrast_level: float = 0.2
45    theme_color_variant: Variant = Variant.VIBRANT
46    captions_provider: CaptionerProvider = CaptionerProvider.BLIP_INSTRUCT
47    llm_provider: LLMProvider = LLMProvider.MINICPM
48    embedder_provider: EmbedderProvider = EmbedderProvider.OPEN_CLIP
49    enable_text_summary: bool = False
50    enable_document_summary: bool = False
51    document_detection_threshold: int = 65
52    face_detection_threshold: float = 0.7
53    enabled_file_modules: set[FileModule] = field(
54        default_factory=lambda: {
55            FileModule.DATA_URL,
56            FileModule.EXIF,
57            FileModule.GPS,
58            FileModule.TAGS,
59            FileModule.TIME,
60            FileModule.WEATHER,
61        }
62    )
63    enabled_visual_modules: set[VisualModule] = field(
64        default_factory=lambda: {
65            VisualModule.CAPTION,
66            VisualModule.EMBEDDING,
67            VisualModule.FACES,
68            VisualModule.OBJECTS,
69            VisualModule.OCR,
70            VisualModule.QUALITY_DETECTION,
71            VisualModule.SUMMARY,
72            VisualModule.COLOR,
73        }
74    )

 5class FileModule(StrEnum):
 6    """Enum for selecting file-based analyzer modules."""
 7
 8    DATA_URL = "DataUrlModule"
 9    EXIF = "ExifModule"
10    GPS = "GPSModule"
11    TAGS = "TagsModule"
12    TIME = "TimeModule"
13    WEATHER = "WeatherModule"

16class VisualModule(StrEnum):
17    """Enum for selecting visual based analyzer modules."""
18
19    CAPTION = "CaptionModule"
20    EMBEDDING = "EmbeddingModule"
21    FACES = "FacesModule"
22    OBJECTS = "ObjectsModule"
23    OCR = "OCRModule"
24    QUALITY_DETECTION = "QualityDetectionModule"
25    SUMMARY = "SummaryModule"
26    COLOR = "ColorModule"

167@dataclass
168class ImageDataOutput:
169    """Comprehensive data for an image.
170
171    Attributes:
172        path: The file system path to the image.
173        exif: Exif data of the image.
174        data_url: The data URL representation of the image.
175        gps: GPS data associated with the image.
176        time: Time-related data for the image.
177        weather: Weather data at the time the image was taken.
178    """
179
180    path: Path
181    exif: ExifData | None = None
182    data_url: str | None = None
183    gps: GPSData | None = None
184    time: TimeData | IntermediateTimeData | None = None
185    weather: WeatherData | None = None
186    tags: TagData | None = None

118@dataclass
119class FrameDataOutput:
120    """Data for a frame.
121
122    Attributes:
123        ocr: The OCR data.
124        embedding: The embedding data.
125        faces: The face boxes.
126        summary: The frame summary.
127        caption_data: Info extracted using caption instructions.
128        objects: The object boxes.
129        measured_quality: The measured quality data.
130    """
131
132    ocr: OCRData | None = None
133    embedding: list[float] | None = None
134    faces: list[FaceBox] | None = None
135    summary: str | None = None
136    caption_data: CaptionData | None = None
137    objects: list[ObjectBox] | None = None
138    measured_quality: MeasuredQualityData | None = None
139    color: ColorData | None = None

11@dataclass
12class ExifData:
13    """Exif Data of the image.
14
15    Attributes:
16        width: The width of the image.
17        height: The height of the image.
18        duration: The duration of the media, if applicable.
19        size_bytes: The size of the file in bytes.
20        format: The format of the image.
21        exif_tool: The output from ExifTool.
22        file: File-related information.
23        composite: Composite data.
24        exif: Exif metadata, if available.
25        xmp: XMP metadata, if available.
26        mpf: Motion photo metadata, if available.
27        jfif: JFIF metadata, if available.
28        icc_profile: ICC profile data, if available.
29        gif: GIF-specific data, if available.
30        quicktime: QuickTime-specific data, if available.
31        matroska: Matroska-specific data, if available.
32    """
33
34    width: int
35    height: int
36    duration: float | None
37    size_bytes: int
38    format: str
39    exif_tool: dict[str, Any]
40    file: dict[str, Any]
41    composite: dict[str, Any]
42    exif: dict[str, Any] | None
43    xmp: dict[str, Any] | None
44    mpf: dict[str, Any] | None
45    jfif: dict[str, Any] | None
46    icc_profile: dict[str, Any] | None
47    gif: dict[str, Any] | None
48    png: dict[str, Any] | None
49    quicktime: dict[str, Any] | None
50    matroska: dict[str, Any] | None

53@dataclass
54class GPSData:
55    """GPS Data related to the image.
56
57    Attributes:
58        latitude: The latitude coordinate.
59        longitude: The longitude coordinate.
60        altitude: The altitude information.
61        location: The geolocation information.
62    """
63
64    latitude: float | None = None
65    longitude: float | None = None
66    altitude: float | None = None
67    location: GeoLocation | None = None

77@dataclass
78class TimeData:
79    """Time-related data for the image.
80
81    Attributes:
82        datetime_local: The local datetime.
83        datetime_source: The source of the datetime information.
84        timezone_name: The name of the timezone.
85        timezone_offset: The offset of the timezone.
86        datetime_utc: The UTC datetime based of the GPS data.
87    """
88
89    datetime_local: datetime
90    datetime_source: str
91    timezone_name: str | None
92    timezone_offset: timedelta | None
93    datetime_utc: datetime | None = None

 96@dataclass
 97class WeatherData:
 98    """Weather data from the time and place the image was taken.
 99
100    Attributes:
101        weather_recorded_at: The datetime when the weather was recorded.
102        weather_temperature: The temperature at the time of recording.
103        weather_dewpoint: The dew point at the time of recording.
104        weather_relative_humidity: The relative humidity at the time of recording.
105        weather_precipitation: The precipitation level at the time of recording.
106        weather_wind_gust: The wind gust speed at the time of recording.
107        weather_pressure: The atmospheric pressure at the time of recording.
108        weather_sun_hours: The sun hours at the time of recording.
109        weather_condition: The weather condition at the time of recording.
110    """
111
112    weather_recorded_at: datetime | None = None
113    weather_temperature: float | None = None
114    weather_dewpoint: float | None = None
115    weather_relative_humidity: float | None = None
116    weather_precipitation: float | None = None
117    weather_wind_gust: float | None = None
118    weather_pressure: float | None = None
119    weather_sun_hours: float | None = None
120    weather_condition: WeatherCondition | None = None

70@dataclass
71class IntermediateTimeData:
72    """Intermediate Time Data related to the image, storing just datetime_utc."""
73
74    datetime_utc: datetime | None = None

123@dataclass
124class TagData:
125    """Tags, such as is_panorama, is_motion_photo, is_night_sight."""
126
127    use_panorama_viewer: bool
128    is_photosphere: bool
129    projection_type: str | None
130    is_motion_photo: bool
131    motion_photo_presentation_timestamp: int | None
132    is_night_sight: bool
133    is_hdr: bool
134    is_burst: bool
135    burst_id: str | None
136    is_timelapse: bool
137    is_slowmotion: bool
138    is_video: bool
139    capture_fps: float | None
140    video_fps: float | None

43@dataclass
44class OCRData:
45    """OCR data for a frame.
46
47    Attributes:
48        has_legible_text: Whether the text is legible.
49        ocr_text: The OCR text.
50        document_summary: The document summary.
51        ocr_boxes: The OCR boxes.
52    """
53
54    has_legible_text: bool
55    ocr_text: str | None
56    document_summary: str | None
57    ocr_boxes: list[OCRBox]

11@dataclass
12class CaptionData:
13    """A model to store structured information about an image."""
14
15    default_caption: str
16    main_subject: str
17    is_indoor: bool
18    contains_pets: bool
19    is_food_or_drink: bool
20    contains_vehicle: bool
21    setting: str
22    is_event: bool
23    contains_landmarks: bool
24    is_document: bool
25    contains_people: bool
26    is_landscape: bool | None = None
27    is_cityscape: bool | None = None
28    pet_type: str | None = None
29    contains_animals: bool | None = None
30    animal_type: str | None = None
31    food_or_drink_type: str | None = None
32    vehicle_type: str | None = None
33    event_type: str | None = None
34    landmark_name: str | None = None
35    document_type: str | None = None
36    people_count: int | None = None
37    people_mood: str | None = None
38    photo_type: str | None = None
39    is_activity: bool | None = None
40    activity_description: str | None = None

60@dataclass
61class MeasuredQualityData:
62    """Measured quality data for a frame.
63
64    Attributes:
65        measured_sharpness: The measured sharpness.
66        measured_noise: The measured noise.
67        measured_brightness: The measured brightness.
68        measured_contrast: The measured contrast.
69        measured_clipping: The measured clipping.
70        measured_dynamic_range: The measured dynamic range.
71        quality_score: The quality score.
72    """
73
74    measured_sharpness: float
75    measured_noise: int
76    measured_brightness: float
77    measured_contrast: float
78    measured_clipping: float
79    measured_dynamic_range: float
80    quality_score: float

 98@dataclass
 99class ColorData:
100    """Color info, and theme generated based on image.
101
102    Attributes:
103        themes: Generated themes based of prominent colors in the image.
104        prominent_colors: Prominent colors extracted from the image.
105        average_hue: Average hue value in degrees.
106        average_saturation: Average saturation value [0 to 100].
107        average_lightness: Average lightness value [0 to 100].
108    """
109
110    themes: list[dict[str, Any]]
111    prominent_colors: list[str]
112    average_hue: float
113    average_saturation: float
114    average_lightness: float
115    histogram: ColorHistogram

91class ColorHistogram(TypedDict):
92    """Types for histogram dict in ColorData."""
93
94    bins: int
95    channels: RGBChannels

 5@dataclass
 6class GeoLocation:
 7    """Represents a reverse geocoded location where a photo/video was taken.
 8
 9    Attributes:
10        country: The country name.
11        city: The city name.
12        province: The province or state name, if applicable.
13        place_latitude: The latitude coordinate of the location.
14        place_longitude: The longitude coordinate of the location.
15    """
16
17    country: str
18    city: str
19    province: str | None
20    place_latitude: float
21    place_longitude: float

18@dataclass
19class ChatMessage:
20    """Chat message dataclass."""
21
22    message: str
23    images: list[Image] = field(default_factory=list)
24    role: ChatRole = ChatRole.USER

10class ChatRole(StrEnum):
11    """Chat roles enum."""
12
13    ASSISTANT = auto()
14    USER = auto()
15    SYSTEM = auto()

13def get_llm_by_provider(provider: LLMProvider) -> BaseVisualLLM:
14    """Get the LLM by the provider."""
15    return llm_providers[provider]()

12class LLMProvider(StrEnum):
13    """LLM providers enum."""
14
15    MINICPM = auto()
16    OPENAI = auto()

 9def get_captioner_by_provider(provider: CaptionerProvider) -> CaptionerProtocol:
10    """Get the captioner by the provider.
11
12    Args:
13        provider: The captioner provider.
14
15    Returns:
16        The captioner.
17    """
18    return {
19        CaptionerProvider.MINICPM: lambda: LLMCaptioner(LLMProvider.MINICPM),
20        CaptionerProvider.OPENAI: lambda: LLMCaptioner(LLMProvider.OPENAI),
21        CaptionerProvider.BLIP: BlipCaptioner,
22        CaptionerProvider.BLIP_INSTRUCT: InstructBlipCaptioner,
23    }[provider]()

19class CaptionerProvider(StrEnum):
20    """Captioner providers enum."""
21
22    BLIP_INSTRUCT = auto()
23    MINICPM = auto()
24    OPENAI = auto()
25    BLIP = auto()

18class PipelineModule(ABC, Generic[TData]):
19    """A generic pipeline module that can process either File-based or Visual data."""
20
21    run_times: list[float]
22    id: str
23    depends: ClassVar[set[AnalyzerModule]] = set()
24
25    def __init__(self) -> None:
26        """Initializes the PipelineModule."""
27        self.id = self.__class__.__name__
28        self.run_times = []
29
30    def run(self, data: TData, config: FullAnalyzerConfig) -> None:
31        """Runs the pipeline module.
32
33        Measuring the execution time and delegating the
34        actual processing to the `process` method.
35
36        Args:
37            data: The data to be processed (ImageData or FrameData).
38            config: The configuration object (e.g., FullAnalyzerConfig).
39        """
40        start_time = time.time()
41
42        self.process(data, config)
43        self.run_times.append(time.time() - start_time)
44
45    @abstractmethod
46    def process(self, data: TData, config: FullAnalyzerConfig) -> None:
47        """Abstract method for processing data. This should be implemented by subclasses.
48
49        Args:
50            data: The data to be processed (ImageData or FrameData).
51            config: The configuration object.
52        """

13class DataUrlModule(PipelineModule[ImageData]):
14    """Convert an image to a data URL."""
15
16    def process(self, data: ImageData, _: FullAnalyzerConfig) -> None:
17        """Convert an image to a data URL."""
18        tiny_height = 6
19        with PIL.Image.open(data.frames[0]) as pil_image:
20            img = pil_image.resize(
21                (
22                    int(pil_image.width / pil_image.height * tiny_height),
23                    tiny_height,
24                ),
25            )
26            buffered = BytesIO()
27            img.save(buffered, format="PNG")
28        data.data_url = base64.b64encode(buffered.getvalue()).decode()

 44class ExifModule(PipelineModule[ImageData]):
 45    """Extract EXIF data from an image using exiftool."""
 46
 47    def process(self, data: ImageData, _: FullAnalyzerConfig) -> None:
 48        """Extract EXIF data from an image."""
 49        with ExifToolHelper() as et:
 50            result = et.execute_json(str(data.path))
 51            exif_dict = structure_exiftool_dict(result[0])
 52            if (
 53                "Composite" not in exif_dict
 54                or "File" not in exif_dict
 55                or "ExifTool" not in exif_dict
 56            ):
 57                raise ValueError(f"Media-analyzer does not support this file {data.path}")
 58
 59        if "EXIF" in exif_dict:
 60            alt_ref = exif_dict["EXIF"].get("GPSAltitudeRef")
 61            # altitude ref = 0 means above sea level
 62            # ref = 1 means below sea level
 63            # LG G4 produces ref = 1.8 for some reason when above sea level
 64            #   (maybe also below?)
 65            if alt_ref not in {0, 1, None}:
 66                if "GPSAltitude" in exif_dict["Composite"]:
 67                    exif_dict["Composite"]["GPSAltitude"] = abs(
 68                        exif_dict["Composite"]["GPSAltitude"],
 69                    )
 70                exif_dict["EXIF"]["GPSAltitudeRef"] = 0
 71
 72        assert "ExifTool" in exif_dict
 73        assert "File" in exif_dict
 74        assert "Composite" in exif_dict
 75        width = exif_dict["File"].get("ImageWidth")
 76        height = exif_dict["File"].get("ImageHeight")
 77        duration: float | None = None
 78        if "GIF" in exif_dict:
 79            width = exif_dict["GIF"]["ImageWidth"]
 80            height = exif_dict["GIF"]["ImageHeight"]
 81        if "PNG" in exif_dict:
 82            width = exif_dict["PNG"]["ImageWidth"]
 83            height = exif_dict["PNG"]["ImageHeight"]
 84        if "QuickTime" in exif_dict:
 85            duration = exif_dict["QuickTime"]["Duration"]
 86            width = exif_dict["QuickTime"]["ImageWidth"]
 87            height = exif_dict["QuickTime"]["ImageHeight"]
 88        if "Matroska" in exif_dict:
 89            width = exif_dict["Matroska"]["ImageWidth"]
 90            height = exif_dict["Matroska"]["ImageHeight"]
 91            duration = parse_duration(exif_dict["Matroska"]["Duration"])
 92
 93        assert width and height
 94        data.exif = ExifData(
 95            size_bytes=exif_dict["File"]["FileSize"],
 96            width=width,
 97            height=height,
 98            duration=duration,
 99            format=exif_dict["File"]["MIMEType"],
100            exif_tool=exif_dict["ExifTool"],
101            file=exif_dict["File"],
102            exif=exif_dict.get("EXIF"),
103            xmp=exif_dict.get("XMP"),
104            mpf=exif_dict.get("MPF"),
105            jfif=exif_dict.get("JFIF"),
106            icc_profile=exif_dict.get("ICC_Profile"),
107            composite=exif_dict["Composite"],
108            gif=exif_dict.get("GIF"),
109            png=exif_dict.get("PNG"),
110            quicktime=exif_dict.get("QuickTime"),
111            matroska=exif_dict.get("Matroska"),
112        )