Initial upload: aesthetic-immich-starter

This commit is contained in:
Abderrahim-Mouhcine
2026-08-16 19:59:33 +01:00
commit 8938106e99
4777 changed files with 2176899 additions and 0 deletions
@@ -0,0 +1,13 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,53 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Tasks Components Containers API."""
import mediapipe.tasks.python.components.containers.audio_data
import mediapipe.tasks.python.components.containers.bounding_box
import mediapipe.tasks.python.components.containers.category
import mediapipe.tasks.python.components.containers.classification_result
import mediapipe.tasks.python.components.containers.detections
import mediapipe.tasks.python.components.containers.embedding_result
import mediapipe.tasks.python.components.containers.landmark
import mediapipe.tasks.python.components.containers.landmark_detection_result
import mediapipe.tasks.python.components.containers.rect
AudioDataFormat = audio_data.AudioDataFormat
AudioData = audio_data.AudioData
BoundingBox = bounding_box.BoundingBox
Category = category.Category
Classifications = classification_result.Classifications
ClassificationResult = classification_result.ClassificationResult
Detection = detections.Detection
DetectionResult = detections.DetectionResult
Embedding = embedding_result.Embedding
EmbeddingResult = embedding_result.EmbeddingResult
Landmark = landmark.Landmark
NormalizedLandmark = landmark.NormalizedLandmark
LandmarksDetectionResult = landmark_detection_result.LandmarksDetectionResult
Rect = rect.Rect
NormalizedRect = rect.NormalizedRect
# Remove unnecessary modules to avoid duplication in API docs.
del audio_data
del bounding_box
del category
del classification_result
del detections
del embedding_result
del landmark
del landmark_detection_result
del rect
del mediapipe
@@ -0,0 +1,152 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe audio data."""
import ctypes
import dataclasses
from typing import Optional
import numpy as np
from mediapipe.tasks.python.components.containers import audio_data_c
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class AudioDataFormat:
"""Audio format metadata.
Attributes:
num_channels: the number of channels of the audio data.
sample_rate: the audio sample rate.
"""
num_channels: int = 1
sample_rate: Optional[float] = None
class AudioData(object):
"""MediaPipe Tasks' audio container."""
def __init__(
self,
buffer_length: int,
audio_format: AudioDataFormat = AudioDataFormat()
) -> None:
"""Initializes the `AudioData` object.
Args:
buffer_length: the length of the audio buffer.
audio_format: the audio format metadata.
"""
self._audio_format = audio_format
self._buffer = np.zeros([buffer_length, self._audio_format.num_channels],
dtype=np.float32)
def clear(self):
"""Clears the internal buffer and fill it with zeros."""
self._buffer.fill(0)
def load_from_array(self,
src: np.ndarray,
offset: int = 0,
size: int = -1) -> None:
"""Loads the audio data from a NumPy array.
Args:
src: A NumPy source array contains the input audio.
offset: An optional offset for loading a slice of the `src` array to the
buffer.
size: An optional size parameter denoting the number of samples to load
from the `src` array.
Raises:
ValueError: If the input array has an incorrect shape or if
`offset` + `size` exceeds the length of the `src` array.
"""
if len(src.shape) == 1:
if self._audio_format.num_channels != 1:
raise ValueError(f"Input audio is mono, but the audio data is expected "
f"to have {self._audio_format.num_channels} channels.")
elif src.shape[1] != self._audio_format.num_channels:
raise ValueError(f"Input audio contains an invalid number of channels. "
f"Expect {self._audio_format.num_channels}.")
if size < 0:
size = len(src)
if offset + size > len(src):
raise ValueError(
f"Index out of range. offset {offset} + size {size} should be <= "
f"src's length: {len(src)}")
if len(src) >= len(self._buffer):
# If the internal buffer is shorter than the load target (src), copy
# values from the end of the src array to the internal buffer.
new_offset = offset + size - len(self._buffer)
new_size = len(self._buffer)
self._buffer = src[new_offset:new_offset + new_size].copy()
else:
# Shift the internal buffer backward and add the incoming data to the end
# of the buffer.
shift = size
self._buffer = np.roll(self._buffer, -shift, axis=0)
self._buffer[-shift:, :] = src[offset:offset + size].copy()
@classmethod
def create_from_array(cls,
src: np.ndarray,
sample_rate: Optional[float] = None) -> "AudioData":
"""Creates an `AudioData` object from a NumPy array.
Args:
src: A NumPy source array contains the input audio.
sample_rate: the optional audio sample rate.
Returns:
An `AudioData` object that contains a copy of the NumPy source array as
the data.
"""
src = src.astype(np.float32)
obj = cls(
buffer_length=src.shape[0],
audio_format=AudioDataFormat(
num_channels=1 if len(src.shape) == 1 else src.shape[1],
sample_rate=sample_rate))
obj.load_from_array(src)
return obj
@property
def audio_format(self) -> AudioDataFormat:
"""Gets the audio format of the audio."""
return self._audio_format
@property
def buffer_length(self) -> int:
"""Gets the sample count of the audio."""
return self._buffer.shape[0]
@property
def buffer(self) -> np.ndarray:
"""Gets the internal buffer."""
return self._buffer
@doc_controls.do_not_generate_docs
def to_ctypes(self) -> audio_data_c.AudioDataC:
"""Converts the object to a ctypes audio data object."""
return audio_data_c.AudioDataC(
num_channels=self.audio_format.num_channels,
sample_rate=self.audio_format.sample_rate,
audio_data=self._buffer.ctypes.data_as(ctypes.POINTER(ctypes.c_float)),
audio_data_size=self._buffer.size,
)
@@ -0,0 +1,27 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe audio data ctypes."""
import ctypes
class AudioDataC(ctypes.Structure):
"""The audio data used in the C API."""
_fields_ = [
('num_channels', ctypes.c_int),
('sample_rate', ctypes.c_double),
('audio_data', ctypes.POINTER(ctypes.c_float)),
('audio_data_size', ctypes.c_size_t),
]
@@ -0,0 +1,66 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Bounding box data class."""
import dataclasses
from typing import Any
from mediapipe.tasks.python.components.containers import rect_c as rect_c_lib
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class BoundingBox:
"""An integer bounding box, axis aligned.
Attributes:
origin_x: The X coordinate of the top-left corner, in pixels.
origin_y: The Y coordinate of the top-left corner, in pixels.
width: The width of the bounding box, in pixels.
height: The height of the bounding box, in pixels.
"""
origin_x: int
origin_y: int
width: int
height: int
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(cls, c_obj: rect_c_lib.RectC) -> 'BoundingBox':
"""Creates a `BoundingBox` object from a ctypes RectC struct."""
return BoundingBox(
origin_x=c_obj.left,
origin_y=c_obj.top,
width=c_obj.right - c_obj.left,
height=c_obj.bottom - c_obj.top,
)
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, BoundingBox):
return False
return (
self.origin_x == other.origin_x
and self.origin_y == other.origin_y
and self.width == other.width
and self.height == other.height
)
@@ -0,0 +1,97 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Category data class."""
import dataclasses
from typing import Any, Optional
from mediapipe.tasks.python.components.containers import category_c as category_c_module
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class Category:
"""A classification category.
Category is a util class, contains a label, its display name, a float
value as score, and the index of the label in the corresponding label file.
Typically it's used as the result of classification tasks.
Attributes:
index: The index of the label in the corresponding label file.
score: The probability score of this label category.
display_name: The display name of the label, which may be translated for
different locales. For example, a label, "apple", may be translated into
Spanish for display purpose, so that the `display_name` is "manzana".
category_name: The label of this category object.
"""
index: Optional[int] = None
score: Optional[float] = None
display_name: Optional[str] = None
category_name: Optional[str] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(cls, c_obj: category_c_module.CategoryC) -> "Category":
"""Creates a `Category` object from the given `CategoryC` object.
This function converts the `CategoryC` index of -1 to a Python value of None
to retain the same semantic meaning. All other values are converted as-is.
Args:
c_obj: The `CategoryC` object to be converted.
Returns:
A `Category` object.
"""
return Category(
index=c_obj.index if c_obj.index != -1 else None,
score=c_obj.score,
category_name=(
c_obj.category_name.decode("utf-8") if c_obj.category_name else None
),
display_name=(
c_obj.display_name.decode("utf-8") if c_obj.display_name else None
),
)
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, Category):
return False
return (
self.index == other.index
and self.score == other.score
and self.display_name == other.display_name
and self.category_name == other.category_name
)
@doc_controls.do_not_generate_docs
def create_list_of_categories_from_ctypes(
c_obj: category_c_module.CategoriesC,
) -> list[Category]:
"""Creates a list of `Category` objects from a `CategoriesC` object."""
return [
Category.from_ctypes(c_obj.categories[i])
for i in range(c_obj.categories_count)
]
@@ -0,0 +1,37 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""C types for Category."""
import ctypes
class CategoryC(ctypes.Structure):
"""The ctypes struct for Category."""
_fields_ = [
("index", ctypes.c_int),
("score", ctypes.c_float),
("category_name", ctypes.c_char_p),
("display_name", ctypes.c_char_p),
]
class CategoriesC(ctypes.Structure):
"""C types for a list of Categories."""
_fields_ = [
("categories", ctypes.POINTER(CategoryC)),
("categories_count", ctypes.c_uint32),
]
@@ -0,0 +1,99 @@
# Copyright 2022 The TensorFlow Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Classifications data class."""
import dataclasses
from typing import List, Optional
from mediapipe.tasks.python.components.containers import category as category_lib
from mediapipe.tasks.python.components.containers import classification_result_c
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class Classifications:
"""Represents the classification results for a given classifier head.
Attributes:
categories: The array of predicted categories, usually sorted by descending
scores (e.g. from high to low probability).
head_index: The index of the classifier head these categories refer to. This
is useful for multi-head models.
head_name: The name of the classifier head, which is the corresponding
tensor metadata name.
"""
categories: List[category_lib.Category]
head_index: int
head_name: Optional[str] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(
cls, c_struct: classification_result_c.ClassificationsC
) -> 'Classifications':
"""Creates a `Classifications` object from the given ctypes struct."""
if c_struct.categories and c_struct.categories_count > 0:
categories = [
category_lib.Category.from_ctypes(c_struct.categories[i])
for i in range(c_struct.categories_count)
]
else:
categories = []
return Classifications(
categories=categories,
head_index=c_struct.head_index,
head_name=(
c_struct.head_name.decode('utf-8') if c_struct.head_name else None
),
)
@dataclasses.dataclass
class ClassificationResult:
"""Contains the classification results of a model.
Attributes:
classifications: A list of `Classifications` objects, each for a head of the
model.
timestamp_ms: The optional timestamp (in milliseconds) of the start of the
chunk of data corresponding to these results. This is only used for
classification on time series (e.g. audio classification). In these use
cases, the amount of data to process might exceed the maximum size that
the model can process: to solve this, the input data is split into
multiple chunks starting at different timestamps.
"""
classifications: List[Classifications]
timestamp_ms: Optional[int] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(
cls, c_struct: classification_result_c.ClassificationResultC
) -> 'ClassificationResult':
"""Creates a `ClassificationResult` object from a ctypes struct."""
if c_struct.classifications and c_struct.classifications_count > 0:
classifications = [
Classifications.from_ctypes(c_struct.classifications[i])
for i in range(c_struct.classifications_count)
]
else:
classifications = []
timestamp_ms = c_struct.timestamp_ms if c_struct.has_timestamp_ms else None
return ClassificationResult(
classifications=classifications, timestamp_ms=timestamp_ms
)
@@ -0,0 +1,37 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""C types for ClassificationResult."""
import ctypes
from mediapipe.tasks.python.components.containers import category_c
class ClassificationsC(ctypes.Structure):
_fields_ = [
("categories", ctypes.POINTER(category_c.CategoryC)),
("categories_count", ctypes.c_uint32),
("head_index", ctypes.c_int),
("head_name", ctypes.c_char_p),
]
class ClassificationResultC(ctypes.Structure):
_fields_ = [
("classifications", ctypes.POINTER(ClassificationsC)),
("classifications_count", ctypes.c_uint32),
("timestamp_ms", ctypes.c_int64),
("has_timestamp_ms", ctypes.c_bool),
]
@@ -0,0 +1,122 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Detections data class."""
import dataclasses
from typing import Any, List, Optional
from mediapipe.tasks.python.components.containers import bounding_box as bounding_box_lib
from mediapipe.tasks.python.components.containers import category as category_lib
from mediapipe.tasks.python.components.containers import category_c as category_c_lib
from mediapipe.tasks.python.components.containers import detections_c as detections_c_lib
from mediapipe.tasks.python.components.containers import keypoint as keypoint_lib
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class Detection:
"""Represents one detected object in the object detector's results.
Attributes:
bounding_box: A BoundingBox object.
categories: A list of Category objects.
keypoints: A list of NormalizedKeypoint objects.
"""
bounding_box: bounding_box_lib.BoundingBox
categories: List[category_lib.Category]
keypoints: Optional[List[keypoint_lib.NormalizedKeypoint]] = None
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, Detection):
return False
return (
self.bounding_box == other.bounding_box
and self.categories == other.categories
and self.keypoints == other.keypoints
)
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(cls, c_obj: detections_c_lib.DetectionC) -> 'Detection':
"""Creates a `Detection` object from the given `DetectionC` object."""
c_categories = category_c_lib.CategoriesC(
categories=c_obj.categories, categories_count=c_obj.categories_count
)
py_categories = category_lib.create_list_of_categories_from_ctypes(
c_categories
)
py_bounding_box = bounding_box_lib.BoundingBox.from_ctypes(
c_obj.bounding_box
)
if c_obj.keypoints and c_obj.keypoints_count > 0:
py_keypoints = [
keypoint_lib.NormalizedKeypoint.from_ctypes(c_obj.keypoints[i])
for i in range(c_obj.keypoints_count)
]
else:
py_keypoints = None
return Detection(
bounding_box=py_bounding_box,
categories=py_categories,
keypoints=py_keypoints,
)
@dataclasses.dataclass
class DetectionResult:
"""Represents the list of detected objects.
Attributes:
detections: A list of `Detection` objects.
"""
detections: List[Detection]
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, DetectionResult):
return False
return self.detections == other.detections
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(
cls, c_obj: detections_c_lib.DetectionResultC
) -> 'DetectionResult':
"""Creates a `DetectionResult` object from a `DetectionResultC` object."""
return DetectionResult(
detections=[
Detection.from_ctypes(c_obj.detections[i])
for i in range(c_obj.detections_count)
]
)
@@ -0,0 +1,41 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Detection Result C API types."""
import ctypes
from mediapipe.tasks.python.components.containers import category_c
from mediapipe.tasks.python.components.containers import keypoint_c
from mediapipe.tasks.python.components.containers import rect_c
class DetectionC(ctypes.Structure):
"""CTypes for a single detection."""
_fields_ = [
('categories', ctypes.POINTER(category_c.CategoryC)),
('categories_count', ctypes.c_uint32),
('bounding_box', rect_c.RectC),
('keypoints', ctypes.POINTER(keypoint_c.NormalizedKeypointC)),
('keypoints_count', ctypes.c_uint32),
]
class DetectionResultC(ctypes.Structure):
"""CTypes for the detection result."""
_fields_ = [
('detections', ctypes.POINTER(DetectionC)),
('detections_count', ctypes.c_uint32),
]
@@ -0,0 +1,93 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Embeddings data class."""
import dataclasses
from typing import List, Optional
import numpy as np
from mediapipe.tasks.python.components.containers import embedding_result_c
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class Embedding:
"""Embedding result for a given embedder head.
Attributes:
embedding: The actual embedding, either floating-point or scalar-quantized.
head_index: The index of the embedder head that produced this embedding.
This is useful for multi-head models.
head_name: The name of the embedder head, which is the corresponding tensor
metadata name (if any). This is useful for multi-head models.
"""
embedding: np.ndarray
head_index: Optional[int] = None
head_name: Optional[str] = None
@dataclasses.dataclass
class EmbeddingResult:
"""Embedding results for a given embedder model.
Attributes:
embeddings: A list of `Embedding` objects.
timestamp_ms: The optional timestamp (in milliseconds) of the start of the
chunk of data corresponding to these results. This is only used for
embedding extraction on time series (e.g. audio embedding). In these use
cases, the amount of data to process might exceed the maximum size that
the model can process: to solve this, the input data is split into
multiple chunks starting at different timestamps.
"""
embeddings: List[Embedding]
timestamp_ms: Optional[int] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(
cls, c_result: embedding_result_c.EmbeddingResultC
) -> 'EmbeddingResult':
"""Converts a C EmbeddingResult to a Python EmbeddingResult."""
python_embeddings = []
for i in range(c_result.embeddings_count):
c_embedding = c_result.embeddings[i]
embedding = None
if c_embedding.float_embedding:
embedding_array = list(
c_embedding.float_embedding[: c_embedding.values_count]
)
embedding = np.array(embedding_array, dtype=float)
if c_embedding.quantized_embedding:
embedding_array = list(
c_embedding.quantized_embedding[: c_embedding.values_count]
)
embedding = np.array(embedding_array, dtype=np.uint8)
python_embedding = Embedding(
embedding=embedding,
head_index=c_embedding.head_index,
head_name=(
c_embedding.head_name.decode('utf-8')
if c_embedding.head_name
else None
),
)
python_embeddings.append(python_embedding)
timestamp_ms = c_result.timestamp_ms if c_result.has_timestamp_ms else None
return EmbeddingResult(
embeddings=python_embeddings, timestamp_ms=timestamp_ms
)
@@ -0,0 +1,39 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Embeddings C API types."""
import ctypes
class EmbeddingC(ctypes.Structure):
"""C struct for an embedding."""
_fields_ = [
('float_embedding', ctypes.POINTER(ctypes.c_float)),
('quantized_embedding', ctypes.POINTER(ctypes.c_uint8)),
('values_count', ctypes.c_uint32),
('head_index', ctypes.c_int),
('head_name', ctypes.c_char_p),
]
class EmbeddingResultC(ctypes.Structure):
"""C struct for embedding result."""
_fields_ = [
('embeddings', ctypes.POINTER(EmbeddingC)),
('embeddings_count', ctypes.c_uint32),
('has_timestamp_ms', ctypes.c_bool),
('timestamp_ms', ctypes.c_int64),
]
@@ -0,0 +1,72 @@
# Copyright 2023 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Keypoint data class."""
import dataclasses
from typing import Any, Optional
from mediapipe.tasks.python.components.containers import keypoint_c as keypoint_c_lib
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class NormalizedKeypoint:
"""A normalized keypoint.
Normalized keypoint represents a point in 2D space with x, y coordinates.
x and y are normalized to [0.0, 1.0] by the image width and height
respectively.
Attributes:
x: The x coordinates of the normalized keypoint.
y: The y coordinates of the normalized keypoint.
label: The optional label of the keypoint.
score: The score of the keypoint.
"""
x: Optional[float] = None
y: Optional[float] = None
label: Optional[str] = None
score: Optional[float] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(
cls, c_obj: keypoint_c_lib.NormalizedKeypointC
) -> 'NormalizedKeypoint':
"""Creates a `NormalizedKeypoint` object from a `NormalizedKeypointC."""
return NormalizedKeypoint(
x=c_obj.x,
y=c_obj.y,
label=c_obj.label.decode('utf-8') if c_obj.label else None,
score=c_obj.score,
)
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, NormalizedKeypoint):
return False
return (
self.x == other.x
and self.y == other.y
and self.label == other.label
and self.score == other.score
)
@@ -0,0 +1,27 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Keypoint C API types."""
import ctypes
class NormalizedKeypointC(ctypes.Structure):
"""A keypoint in normalized coordinates."""
_fields_ = [
('x', ctypes.c_float),
('y', ctypes.c_float),
('label', ctypes.c_char_p),
('score', ctypes.c_float),
]
@@ -0,0 +1,108 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Landmark data class."""
import dataclasses
from typing import Optional
from mediapipe.tasks.python.components.containers import landmark_c as landmark_c_module
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class Landmark:
"""A landmark that can have 1 to 3 dimensions.
Use x for 1D points, (x, y) for 2D points and (x, y, z) for 3D points.
Attributes:
x: The x coordinate.
y: The y coordinate.
z: The z coordinate.
visibility: Landmark visibility. Should stay unset if not supported. Float
score of whether landmark is visible or occluded by other objects.
Landmark considered as invisible also if it is not present on the screen
(out of scene bounds). Depending on the model, visibility value is either
a sigmoid or an argument of sigmoid.
presence: Landmark presence. Should stay unset if not supported. Float score
of whether landmark is present on the scene (located within scene bounds).
Depending on the model, presence value is either a result of sigmoid or an
argument of sigmoid function to get landmark presence probability.
name: The name of the landmark.
"""
x: Optional[float] = None
y: Optional[float] = None
z: Optional[float] = None
visibility: Optional[float] = None
presence: Optional[float] = None
name: Optional[str] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(cls, c_struct: landmark_c_module.LandmarkC) -> 'Landmark':
"""Creates a `Landmark` object from the given ctypes struct."""
return Landmark(
x=c_struct.x,
y=c_struct.y,
z=c_struct.z,
visibility=c_struct.visibility if c_struct.has_visibility else None,
presence=c_struct.presence if c_struct.has_presence else None,
name=c_struct.name.decode('utf-8') if c_struct.name else None,
)
@dataclasses.dataclass
class NormalizedLandmark:
"""A normalized version of above Landmark proto.
All coordinates should be within [0, 1].
Attributes:
x: The normalized x coordinate.
y: The normalized y coordinate.
z: The normalized z coordinate.
visibility: Landmark visibility. Should stay unset if not supported. Float
score of whether landmark is visible or occluded by other objects.
Landmark considered as invisible also if it is not present on the screen
(out of scene bounds). Depending on the model, visibility value is either
a sigmoid or an argument of sigmoid.
presence: Landmark presence. Should stay unset if not supported. Float score
of whether landmark is present on the scene (located within scene bounds).
Depending on the model, presence value is either a result of sigmoid or an
argument of sigmoid function to get landmark presence probability.
name: The name of the landmark.
"""
x: Optional[float] = None
y: Optional[float] = None
z: Optional[float] = None
visibility: Optional[float] = None
presence: Optional[float] = None
name: Optional[str] = None
@classmethod
@doc_controls.do_not_generate_docs
def from_ctypes(
cls, c_struct: landmark_c_module.NormalizedLandmarkC
) -> 'NormalizedLandmark':
"""Creates a `NormalizedLandmark` object from the given ctypes struct."""
return NormalizedLandmark(
x=c_struct.x,
y=c_struct.y,
z=c_struct.z,
visibility=c_struct.visibility if c_struct.has_visibility else None,
presence=c_struct.presence if c_struct.has_presence else None,
name=c_struct.name.decode('utf-8') if c_struct.name else None,
)
@@ -0,0 +1,65 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""C types for Landmark."""
import ctypes
class LandmarkC(ctypes.Structure):
"""The ctypes struct for Landmark."""
_fields_ = [
('x', ctypes.c_float),
('y', ctypes.c_float),
('z', ctypes.c_float),
('has_visibility', ctypes.c_bool),
('visibility', ctypes.c_float),
('has_presence', ctypes.c_bool),
('presence', ctypes.c_float),
('name', ctypes.c_char_p),
]
class LandmarksC(ctypes.Structure):
"""The ctypes struct for a list of Landmarks."""
_fields_ = [
('landmarks', ctypes.POINTER(LandmarkC)),
('landmarks_count', ctypes.c_uint32),
]
class NormalizedLandmarkC(ctypes.Structure):
"""The ctypes struct for NormalizedLandmark."""
_fields_ = [
('x', ctypes.c_float),
('y', ctypes.c_float),
('z', ctypes.c_float),
('has_visibility', ctypes.c_bool),
('visibility', ctypes.c_float),
('has_presence', ctypes.c_bool),
('presence', ctypes.c_float),
('name', ctypes.c_char_p),
]
class NormalizedLandmarksC(ctypes.Structure):
"""The ctypes struct for a list of NormalizedLandmarks."""
_fields_ = [
('landmarks', ctypes.POINTER(NormalizedLandmarkC)),
('landmarks_count', ctypes.c_uint32),
]
@@ -0,0 +1,42 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Landmarks Detection Result data class."""
import dataclasses
from typing import Optional, List
from mediapipe.tasks.python.components.containers import category as category_module
from mediapipe.tasks.python.components.containers import landmark as landmark_module
from mediapipe.tasks.python.components.containers import rect as rect_module
_Category = category_module.Category
_NormalizedLandmark = landmark_module.NormalizedLandmark
_Landmark = landmark_module.Landmark
@dataclasses.dataclass
class LandmarksDetectionResult:
"""Represents the landmarks detection result.
Attributes:
landmarks: A list of `NormalizedLandmark` objects.
categories: A list of `Category` objects.
world_landmarks: A list of `Landmark` objects.
rect: A `NormalizedRect` object.
"""
landmarks: Optional[List[_NormalizedLandmark]]
categories: Optional[List[_Category]]
world_landmarks: Optional[List[_Landmark]]
rect: rect_module.NormalizedRect
@@ -0,0 +1,50 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""C types for Matrix."""
import ctypes
import numpy as np
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
class MatrixC(ctypes.Structure):
"""The ctypes struct for Matrix."""
_fields_ = [
('rows', ctypes.c_uint32),
('cols', ctypes.c_uint32),
('data', ctypes.POINTER(ctypes.c_float)),
]
@doc_controls.do_not_generate_docs
def to_numpy(self) -> np.ndarray:
"""Copies the MatrixC struct to a numpy array.
The data is reshaped from column-major order to (cols, rows) and then
transposed to a row-major numpy array. The underlying data is copied to
ensure the data is not freed when the underlying C Matrix is closed.
Returns:
A numpy array representing the MatrixC struct.
"""
if not self.data or self.rows == 0 or self.cols == 0:
return np.empty((0,))
np_array = np.ctypeslib.as_array(
self.data, shape=(self.cols, self.rows)
).copy()
return np_array.T
@@ -0,0 +1,123 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rect data class."""
import dataclasses
from typing import Any, Optional
from mediapipe.tasks.python.components.containers import rect_c as rect_c_module
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
@dataclasses.dataclass
class Rect:
"""A rectangle, used as part of detection results.
Absolute coordinates. The origin is on the top-left corner of the image.
Attributes:
left: The X coordinate of the left side of the rectangle.
top: The Y coordinate of the top of the rectangle.
bottom: The Y coordinate of the bottom of the rectangle.
right: The X coordinate of the right side of the rectangle.
"""
left: int
top: int
bottom: int
right: int
@doc_controls.do_not_generate_docs
def to_ctypes(self) -> rect_c_module.RectC:
"""Generates a C API RectC object."""
return rect_c_module.RectC(
left=self.left, top=self.top, bottom=self.bottom, right=self.right
)
@dataclasses.dataclass
class RectF:
"""A rectangle, used as part as input region-of-interest.
The coordinates are normalized wrt the image dimensions, i.e. generally in
[0,1] but they may exceed these bounds if describing a region overlapping the
image. The origin is on the top-left corner of the image.
Attributes:
left: The X coordinate of the left side of the rectangle.
top: The Y coordinate of the top of the rectangle.
bottom: The Y coordinate of the bottom of the rectangle.
right: The X coordinate of the right side of the rectangle.
"""
left: float
top: float
bottom: float
right: float
@doc_controls.do_not_generate_docs
def to_ctypes(self) -> rect_c_module.RectFC:
"""Generates a C API RectFC object."""
return rect_c_module.RectFC(
left=self.left, top=self.top, bottom=self.bottom, right=self.right
)
@dataclasses.dataclass
class NormalizedRect:
"""A rectangle with rotation in normalized coordinates.
Location of the center of the rectangle in image coordinates. The (0.0, 0.0)
point is at the (top, left) corner.
The values of box center location and size are within [0, 1].
Attributes:
x_center: The normalized X coordinate of the rectangle, in image
coordinates.
y_center: The normalized Y coordinate of the rectangle, in image
coordinates.
width: The width of the rectangle.
height: The height of the rectangle.
rotation: Rotation angle is clockwise in radians.
rect_id: Optional unique id to help associate different rectangles to each
other.
"""
x_center: float
y_center: float
width: float
height: float
rotation: Optional[float] = 0.0
rect_id: Optional[int] = None
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, NormalizedRect):
return False
return (
self.x_center == other.x_center
and self.y_center == other.y_center
and self.width == other.width
and self.height == other.height
and self.rotation == other.rotation
and self.rect_id == other.rect_id
)
@@ -0,0 +1,50 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Rect C API types."""
import ctypes
class RectC(ctypes.Structure):
"""A rectangle, used as part of detection results."""
_fields_ = [
('left', ctypes.c_int),
('top', ctypes.c_int),
('bottom', ctypes.c_int),
('right', ctypes.c_int),
]
class RectFC(ctypes.Structure):
"""A rectangle (operating on floats), used as part of processing options."""
_fields_ = [
('left', ctypes.c_float),
('top', ctypes.c_float),
('bottom', ctypes.c_float),
('right', ctypes.c_float),
]
class NormalizedRectC(ctypes.Structure):
"""A rectangle with rotation in normalized coordinates."""
_fields_ = [
('left', ctypes.c_float),
('top', ctypes.c_float),
('bottom', ctypes.c_float),
('right', ctypes.c_float),
]
@@ -0,0 +1,23 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MediaPipe Tasks Components Processors API."""
import mediapipe.tasks.python.components.processors.classifier_options
ClassifierOptions = classifier_options.ClassifierOptions
# Remove unnecessary modules to avoid duplication in API docs.
del classifier_options
del mediapipe
@@ -0,0 +1,64 @@
# Copyright 2022 The TensorFlow Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Classifier options data class."""
import dataclasses
from typing import Any, List, Optional
@dataclasses.dataclass
class ClassifierOptions:
"""Options for classification processor.
Attributes:
display_names_locale: The locale to use for display names specified through
the TFLite Model Metadata.
max_results: The maximum number of top-scored classification results to
return.
score_threshold: Overrides the ones provided in the model metadata. Results
below this value are rejected.
category_allowlist: Allowlist of category names. If non-empty, detection
results whose category name is not in this set will be filtered out.
Duplicate or unknown category names are ignored. Mutually exclusive with
`category_denylist`.
category_denylist: Denylist of category names. If non-empty, detection
results whose category name is in this set will be filtered out. Duplicate
or unknown category names are ignored. Mutually exclusive with
`category_allowlist`.
"""
display_names_locale: Optional[str] = None
max_results: Optional[int] = None
score_threshold: Optional[float] = None
category_allowlist: Optional[List[str]] = None
category_denylist: Optional[List[str]] = None
def __eq__(self, other: Any) -> bool:
"""Checks if this object is equal to the given object.
Args:
other: The object to be compared with.
Returns:
True if the objects are equal.
"""
if not isinstance(other, ClassifierOptions):
return False
return (
self.display_names_locale == other.display_names_locale
and self.max_results == other.max_results
and self.score_threshold == other.score_threshold
and self.category_allowlist == other.category_allowlist
and self.category_denylist == other.category_denylist
)
@@ -0,0 +1,81 @@
# Copyright 2025 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""C types for ClassifierOptions."""
import ctypes
import mediapipe.tasks.python.components.processors.classifier_options as classifier_options_module
class ClassifierOptionsC(ctypes.Structure):
_fields_ = [
("display_names_locale", ctypes.c_char_p),
("max_results", ctypes.c_int),
("score_threshold", ctypes.c_float),
("category_allowlist", ctypes.POINTER(ctypes.c_char_p)),
("category_allowlist_count", ctypes.c_uint32),
("category_denylist", ctypes.POINTER(ctypes.c_char_p)),
("category_denylist_count", ctypes.c_uint32),
]
def convert_to_classifier_options_c(
src: classifier_options_module.ClassifierOptions,
) -> ClassifierOptionsC:
"""Converts a Python ClassifierOptions object to a ClassifierOptionsC object."""
options = ClassifierOptionsC()
if src.display_names_locale is not None:
options.display_names_locale = src.display_names_locale.encode("utf-8")
else:
options.display_names_locale = None
if src.max_results is not None:
options.max_results = src.max_results
else:
options.max_results = -1
if src.score_threshold is not None:
options.score_threshold = src.score_threshold
else:
options.score_threshold = 0.0
# Handle category_allowlist
if src.category_allowlist:
bytes_allowlist = [s.encode("utf-8") for s in src.category_allowlist]
allowlist_array_type = ctypes.c_char_p * len(bytes_allowlist)
allowlist_array = allowlist_array_type(*bytes_allowlist)
options.category_allowlist = ctypes.cast(
allowlist_array, ctypes.POINTER(ctypes.c_char_p)
)
options.category_allowlist_count = len(bytes_allowlist)
else:
options.category_allowlist = None
options.category_allowlist_count = 0
# Handle category_denylist
if src.category_denylist:
bytes_denylist = [s.encode("utf-8") for s in src.category_denylist]
denylist_array_type = ctypes.c_char_p * len(bytes_denylist)
denylist_array = denylist_array_type(*bytes_denylist)
options.category_denylist = ctypes.cast(
denylist_array, ctypes.POINTER(ctypes.c_char_p)
)
options.category_denylist_count = len(bytes_denylist)
else:
options.category_denylist = None
options.category_denylist_count = 0
return options
@@ -0,0 +1,13 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,68 @@
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Cosine similarity utilities."""
import numpy as np
from mediapipe.tasks.python.components.containers import embedding_result
_Embedding = embedding_result.Embedding
def _compute_cosine_similarity(u: np.ndarray, v: np.ndarray):
"""Computes cosine similarity between two embeddings."""
if len(u) <= 0:
raise ValueError("Cannot compute cosing similarity on empty embeddings.")
norm_u = np.linalg.norm(u)
norm_v = np.linalg.norm(v)
if norm_u <= 0 or norm_v <= 0:
raise ValueError(
"Cannot compute cosine similarity on embedding with 0 norm.")
return u.dot(v) / (norm_u * norm_v)
def cosine_similarity(u: _Embedding, v: _Embedding) -> float:
"""Utility function to compute cosine similarity between two embedding.
May return an InvalidArgumentError if e.g. the feature vectors are of
different types (quantized vs. float), have different sizes, or have an
L2-norm of 0.
Args:
u: An embedding.
v: An embedding.
Returns:
Cosine similarity value.
"""
if len(u.embedding) != len(v.embedding):
raise ValueError(f"Cannot compute cosine similarity between embeddings "
f"of different sizes "
f"({len(u.embedding)} vs. {len(v.embedding)}).")
if u.embedding.dtype == float and v.embedding.dtype == float:
return _compute_cosine_similarity(u.embedding, v.embedding)
if u.embedding.dtype == np.uint8 and v.embedding.dtype == np.uint8:
return _compute_cosine_similarity(
u.embedding.view("int8").astype("float"),
v.embedding.view("int8").astype("float"),
)
raise ValueError("Cannot compute cosine similarity between quantized and "
"float embeddings.")