hand
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""MediaPipe Tasks Text API."""
|
||||
|
||||
import mediapipe.tasks.python.text.language_detector
|
||||
import mediapipe.tasks.python.text.text_classifier
|
||||
import mediapipe.tasks.python.text.text_embedder
|
||||
|
||||
LanguageDetector = language_detector.LanguageDetector
|
||||
LanguageDetectorOptions = language_detector.LanguageDetectorOptions
|
||||
LanguageDetectorResult = language_detector.LanguageDetectorResult
|
||||
TextClassifier = text_classifier.TextClassifier
|
||||
TextClassifierOptions = text_classifier.TextClassifierOptions
|
||||
TextClassifierResult = text_classifier.TextClassifierResult
|
||||
TextEmbedder = text_embedder.TextEmbedder
|
||||
TextEmbedderOptions = text_embedder.TextEmbedderOptions
|
||||
TextEmbedderResult = text_embedder.TextEmbedderResult
|
||||
|
||||
# Remove unnecessary modules to avoid duplication in API docs.
|
||||
del mediapipe
|
||||
del language_detector
|
||||
del text_classifier
|
||||
del text_embedder
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,16 @@
|
||||
"""Copyright 2022 The MediaPipe Authors.
|
||||
|
||||
All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+54
@@ -0,0 +1,54 @@
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""MediaPipe text task base api."""
|
||||
|
||||
from mediapipe.framework import calculator_pb2
|
||||
from mediapipe.python._framework_bindings import task_runner
|
||||
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
|
||||
|
||||
_TaskRunner = task_runner.TaskRunner
|
||||
|
||||
|
||||
@doc_controls.do_not_generate_docs
|
||||
class BaseTextTaskApi(object):
|
||||
"""The base class of the user-facing mediapipe text task api classes."""
|
||||
|
||||
def __init__(self,
|
||||
graph_config: calculator_pb2.CalculatorGraphConfig) -> None:
|
||||
"""Initializes the `BaseVisionTaskApi` object.
|
||||
|
||||
Args:
|
||||
graph_config: The mediapipe text task graph config proto.
|
||||
"""
|
||||
self._runner = _TaskRunner.create(graph_config)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Shuts down the mediapipe text task instance.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the mediapipe text task failed to close.
|
||||
"""
|
||||
self._runner.close()
|
||||
|
||||
def __enter__(self):
|
||||
"""Returns `self` upon entering the runtime context."""
|
||||
return self
|
||||
|
||||
def __exit__(self, unused_exc_type, unused_exc_value, unused_traceback):
|
||||
"""Shuts down the mediapipe text task instance on exit of the context manager.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the mediapipe text task failed to close.
|
||||
"""
|
||||
self.close()
|
||||
@@ -0,0 +1,220 @@
|
||||
# Copyright 2023 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""MediaPipe language detector task."""
|
||||
|
||||
import dataclasses
|
||||
from typing import List, Optional
|
||||
|
||||
from mediapipe.python import packet_creator
|
||||
from mediapipe.python import packet_getter
|
||||
from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
|
||||
from mediapipe.tasks.cc.components.processors.proto import classifier_options_pb2
|
||||
from mediapipe.tasks.cc.text.text_classifier.proto import text_classifier_graph_options_pb2
|
||||
from mediapipe.tasks.python.components.containers import classification_result as classification_result_module
|
||||
from mediapipe.tasks.python.core import base_options as base_options_module
|
||||
from mediapipe.tasks.python.core import task_info as task_info_module
|
||||
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
|
||||
from mediapipe.tasks.python.text.core import base_text_task_api
|
||||
|
||||
_ClassificationResult = classification_result_module.ClassificationResult
|
||||
_BaseOptions = base_options_module.BaseOptions
|
||||
_TextClassifierGraphOptionsProto = (
|
||||
text_classifier_graph_options_pb2.TextClassifierGraphOptions
|
||||
)
|
||||
_ClassifierOptionsProto = classifier_options_pb2.ClassifierOptions
|
||||
_TaskInfo = task_info_module.TaskInfo
|
||||
|
||||
_CLASSIFICATIONS_STREAM_NAME = 'classifications_out'
|
||||
_CLASSIFICATIONS_TAG = 'CLASSIFICATIONS'
|
||||
_TEXT_IN_STREAM_NAME = 'text_in'
|
||||
_TEXT_TAG = 'TEXT'
|
||||
_TASK_GRAPH_NAME = 'mediapipe.tasks.text.text_classifier.TextClassifierGraph'
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class LanguageDetectorResult:
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Detection:
|
||||
"""A language code and its probability."""
|
||||
|
||||
# An i18n language / locale code, e.g. "en" for English, "uz" for Uzbek,
|
||||
# "ja"-Latn for Japanese (romaji).
|
||||
language_code: str
|
||||
probability: float
|
||||
|
||||
detections: List[Detection]
|
||||
|
||||
|
||||
def _extract_language_detector_result(
|
||||
classification_result: classification_result_module.ClassificationResult,
|
||||
) -> LanguageDetectorResult:
|
||||
"""Extracts a LanguageDetectorResult from a ClassificationResult."""
|
||||
if len(classification_result.classifications) != 1:
|
||||
raise ValueError(
|
||||
'The LanguageDetector TextClassifierGraph should have exactly one '
|
||||
'classification head.'
|
||||
)
|
||||
languages_and_scores = classification_result.classifications[0]
|
||||
language_detector_result = LanguageDetectorResult([])
|
||||
for category in languages_and_scores.categories:
|
||||
if category.category_name is None:
|
||||
raise ValueError(
|
||||
'LanguageDetector ClassificationResult has a missing language code.'
|
||||
)
|
||||
prediction = LanguageDetectorResult.Detection(
|
||||
category.category_name, category.score
|
||||
)
|
||||
language_detector_result.detections.append(prediction)
|
||||
return language_detector_result
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class LanguageDetectorOptions:
|
||||
"""Options for the language detector task.
|
||||
|
||||
Attributes:
|
||||
base_options: Base options for the language detector task.
|
||||
display_names_locale: The locale to use for display names specified through
|
||||
the TFLite Model Metadata.
|
||||
max_results: The maximum number of top-scored classification results to
|
||||
return.
|
||||
score_threshold: Overrides the ones provided in the model metadata. Results
|
||||
below this value are rejected.
|
||||
category_allowlist: Allowlist of category names. If non-empty,
|
||||
classification results whose category name is not in this set will be
|
||||
filtered out. Duplicate or unknown category names are ignored. Mutually
|
||||
exclusive with `category_denylist`.
|
||||
category_denylist: Denylist of category names. If non-empty, classification
|
||||
results whose category name is in this set will be filtered out. Duplicate
|
||||
or unknown category names are ignored. Mutually exclusive with
|
||||
`category_allowlist`.
|
||||
"""
|
||||
|
||||
base_options: _BaseOptions
|
||||
display_names_locale: Optional[str] = None
|
||||
max_results: Optional[int] = None
|
||||
score_threshold: Optional[float] = None
|
||||
category_allowlist: Optional[List[str]] = None
|
||||
category_denylist: Optional[List[str]] = None
|
||||
|
||||
@doc_controls.do_not_generate_docs
|
||||
def to_pb2(self) -> _TextClassifierGraphOptionsProto:
|
||||
"""Generates an TextClassifierOptions protobuf object."""
|
||||
base_options_proto = self.base_options.to_pb2()
|
||||
classifier_options_proto = _ClassifierOptionsProto(
|
||||
score_threshold=self.score_threshold,
|
||||
category_allowlist=self.category_allowlist,
|
||||
category_denylist=self.category_denylist,
|
||||
display_names_locale=self.display_names_locale,
|
||||
max_results=self.max_results,
|
||||
)
|
||||
|
||||
return _TextClassifierGraphOptionsProto(
|
||||
base_options=base_options_proto,
|
||||
classifier_options=classifier_options_proto,
|
||||
)
|
||||
|
||||
|
||||
class LanguageDetector(base_text_task_api.BaseTextTaskApi):
|
||||
"""Class that predicts the language of an input text.
|
||||
|
||||
This API expects a TFLite model with TFLite Model Metadata that contains the
|
||||
mandatory (described below) input tensors, output tensor, and the language
|
||||
codes in an AssociatedFile.
|
||||
|
||||
Input tensors:
|
||||
(kTfLiteString)
|
||||
- 1 input tensor that is scalar or has shape [1] containing the input
|
||||
string.
|
||||
Output tensor:
|
||||
(kTfLiteFloat32)
|
||||
- 1 output tensor of shape`[1 x N]` where `N` is the number of languages.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def create_from_model_path(cls, model_path: str) -> 'LanguageDetector':
|
||||
"""Creates an `LanguageDetector` object from a TensorFlow Lite model and the default `LanguageDetectorOptions`.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model.
|
||||
|
||||
Returns:
|
||||
`LanguageDetector` object that's created from the model file and the
|
||||
default `LanguageDetectorOptions`.
|
||||
|
||||
Raises:
|
||||
ValueError: If failed to create `LanguageDetector` object from the
|
||||
provided
|
||||
file such as invalid file path.
|
||||
RuntimeError: If other types of error occurred.
|
||||
"""
|
||||
base_options = _BaseOptions(model_asset_path=model_path)
|
||||
options = LanguageDetectorOptions(base_options=base_options)
|
||||
return cls.create_from_options(options)
|
||||
|
||||
@classmethod
|
||||
def create_from_options(
|
||||
cls, options: LanguageDetectorOptions
|
||||
) -> 'LanguageDetector':
|
||||
"""Creates the `LanguageDetector` object from language detector options.
|
||||
|
||||
Args:
|
||||
options: Options for the language detector task.
|
||||
|
||||
Returns:
|
||||
`LanguageDetector` object that's created from `options`.
|
||||
|
||||
Raises:
|
||||
ValueError: If failed to create `LanguageDetector` object from
|
||||
`LanguageDetectorOptions` such as missing the model.
|
||||
RuntimeError: If other types of error occurred.
|
||||
"""
|
||||
task_info = _TaskInfo(
|
||||
task_graph=_TASK_GRAPH_NAME,
|
||||
input_streams=[':'.join([_TEXT_TAG, _TEXT_IN_STREAM_NAME])],
|
||||
output_streams=[
|
||||
':'.join([_CLASSIFICATIONS_TAG, _CLASSIFICATIONS_STREAM_NAME])
|
||||
],
|
||||
task_options=options,
|
||||
)
|
||||
return cls(task_info.generate_graph_config())
|
||||
|
||||
def detect(self, text: str) -> LanguageDetectorResult:
|
||||
"""Predicts the language of the input `text`.
|
||||
|
||||
Args:
|
||||
text: The input text.
|
||||
|
||||
Returns:
|
||||
A `LanguageDetectorResult` object that contains a list of languages and
|
||||
scores.
|
||||
|
||||
Raises:
|
||||
ValueError: If any of the input arguments is invalid.
|
||||
RuntimeError: If language detection failed to run.
|
||||
"""
|
||||
output_packets = self._runner.process(
|
||||
{_TEXT_IN_STREAM_NAME: packet_creator.create_string(text)}
|
||||
)
|
||||
|
||||
classification_result_proto = classifications_pb2.ClassificationResult()
|
||||
classification_result_proto.CopyFrom(
|
||||
packet_getter.get_proto(output_packets[_CLASSIFICATIONS_STREAM_NAME])
|
||||
)
|
||||
|
||||
classification_result = _ClassificationResult.create_from_pb2(
|
||||
classification_result_proto
|
||||
)
|
||||
return _extract_language_detector_result(classification_result)
|
||||
@@ -0,0 +1,187 @@
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""MediaPipe text classifier task."""
|
||||
|
||||
import dataclasses
|
||||
from typing import Optional, List
|
||||
|
||||
from mediapipe.python import packet_creator
|
||||
from mediapipe.python import packet_getter
|
||||
from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
|
||||
from mediapipe.tasks.cc.components.processors.proto import classifier_options_pb2
|
||||
from mediapipe.tasks.cc.text.text_classifier.proto import text_classifier_graph_options_pb2
|
||||
from mediapipe.tasks.python.components.containers import classification_result as classification_result_module
|
||||
from mediapipe.tasks.python.core import base_options as base_options_module
|
||||
from mediapipe.tasks.python.core import task_info as task_info_module
|
||||
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
|
||||
from mediapipe.tasks.python.text.core import base_text_task_api
|
||||
|
||||
TextClassifierResult = classification_result_module.ClassificationResult
|
||||
_BaseOptions = base_options_module.BaseOptions
|
||||
_TextClassifierGraphOptionsProto = text_classifier_graph_options_pb2.TextClassifierGraphOptions
|
||||
_ClassifierOptionsProto = classifier_options_pb2.ClassifierOptions
|
||||
_TaskInfo = task_info_module.TaskInfo
|
||||
|
||||
_CLASSIFICATIONS_STREAM_NAME = 'classifications_out'
|
||||
_CLASSIFICATIONS_TAG = 'CLASSIFICATIONS'
|
||||
_TEXT_IN_STREAM_NAME = 'text_in'
|
||||
_TEXT_TAG = 'TEXT'
|
||||
_TASK_GRAPH_NAME = 'mediapipe.tasks.text.text_classifier.TextClassifierGraph'
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TextClassifierOptions:
|
||||
"""Options for the text classifier task.
|
||||
|
||||
Attributes:
|
||||
base_options: Base options for the text classifier task.
|
||||
display_names_locale: The locale to use for display names specified through
|
||||
the TFLite Model Metadata.
|
||||
max_results: The maximum number of top-scored classification results to
|
||||
return.
|
||||
score_threshold: Overrides the ones provided in the model metadata. Results
|
||||
below this value are rejected.
|
||||
category_allowlist: Allowlist of category names. If non-empty,
|
||||
classification results whose category name is not in this set will be
|
||||
filtered out. Duplicate or unknown category names are ignored. Mutually
|
||||
exclusive with `category_denylist`.
|
||||
category_denylist: Denylist of category names. If non-empty, classification
|
||||
results whose category name is in this set will be filtered out. Duplicate
|
||||
or unknown category names are ignored. Mutually exclusive with
|
||||
`category_allowlist`.
|
||||
"""
|
||||
base_options: _BaseOptions
|
||||
display_names_locale: Optional[str] = None
|
||||
max_results: Optional[int] = None
|
||||
score_threshold: Optional[float] = None
|
||||
category_allowlist: Optional[List[str]] = None
|
||||
category_denylist: Optional[List[str]] = None
|
||||
|
||||
@doc_controls.do_not_generate_docs
|
||||
def to_pb2(self) -> _TextClassifierGraphOptionsProto:
|
||||
"""Generates an TextClassifierOptions protobuf object."""
|
||||
base_options_proto = self.base_options.to_pb2()
|
||||
classifier_options_proto = _ClassifierOptionsProto(
|
||||
score_threshold=self.score_threshold,
|
||||
category_allowlist=self.category_allowlist,
|
||||
category_denylist=self.category_denylist,
|
||||
display_names_locale=self.display_names_locale,
|
||||
max_results=self.max_results)
|
||||
|
||||
return _TextClassifierGraphOptionsProto(
|
||||
base_options=base_options_proto,
|
||||
classifier_options=classifier_options_proto)
|
||||
|
||||
|
||||
class TextClassifier(base_text_task_api.BaseTextTaskApi):
|
||||
"""Class that performs classification on text.
|
||||
|
||||
This API expects a TFLite model with (optional) TFLite Model Metadata that
|
||||
contains the mandatory (described below) input tensors, output tensor,
|
||||
and the optional (but recommended) category labels as AssociatedFiles with
|
||||
type
|
||||
TENSOR_AXIS_LABELS per output classification tensor. Metadata is required for
|
||||
models with int32 input tensors because it contains the input process unit
|
||||
for the model's Tokenizer. No metadata is required for models with string
|
||||
input tensors.
|
||||
|
||||
Input tensors:
|
||||
(kTfLiteInt32)
|
||||
- 3 input tensors of size `[batch_size x bert_max_seq_len]` representing
|
||||
the input ids, segment ids, and mask ids
|
||||
- or 1 input tensor of size `[batch_size x max_seq_len]` representing the
|
||||
input ids
|
||||
or (kTfLiteString)
|
||||
- 1 input tensor that is shapeless or has shape [1] containing the input
|
||||
string
|
||||
At least one output tensor with:
|
||||
(kTfLiteFloat32/kBool)
|
||||
- `[1 x N]` array with `N` represents the number of categories.
|
||||
- optional (but recommended) category labels as AssociatedFiles with type
|
||||
TENSOR_AXIS_LABELS, containing one label per line. The first such
|
||||
AssociatedFile (if any) is used to fill the `category_name` field of the
|
||||
results. The `display_name` field is filled from the AssociatedFile (if
|
||||
any) whose locale matches the `display_names_locale` field of the
|
||||
`TextClassifierOptions` used at creation time ("en" by default, i.e.
|
||||
English). If none of these are available, only the `index` field of the
|
||||
results will be filled.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def create_from_model_path(cls, model_path: str) -> 'TextClassifier':
|
||||
"""Creates an `TextClassifier` object from a TensorFlow Lite model and the default `TextClassifierOptions`.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model.
|
||||
|
||||
Returns:
|
||||
`TextClassifier` object that's created from the model file and the
|
||||
default `TextClassifierOptions`.
|
||||
|
||||
Raises:
|
||||
ValueError: If failed to create `TextClassifier` object from the provided
|
||||
file such as invalid file path.
|
||||
RuntimeError: If other types of error occurred.
|
||||
"""
|
||||
base_options = _BaseOptions(model_asset_path=model_path)
|
||||
options = TextClassifierOptions(base_options=base_options)
|
||||
return cls.create_from_options(options)
|
||||
|
||||
@classmethod
|
||||
def create_from_options(cls,
|
||||
options: TextClassifierOptions) -> 'TextClassifier':
|
||||
"""Creates the `TextClassifier` object from text classifier options.
|
||||
|
||||
Args:
|
||||
options: Options for the text classifier task.
|
||||
|
||||
Returns:
|
||||
`TextClassifier` object that's created from `options`.
|
||||
|
||||
Raises:
|
||||
ValueError: If failed to create `TextClassifier` object from
|
||||
`TextClassifierOptions` such as missing the model.
|
||||
RuntimeError: If other types of error occurred.
|
||||
"""
|
||||
task_info = _TaskInfo(
|
||||
task_graph=_TASK_GRAPH_NAME,
|
||||
input_streams=[':'.join([_TEXT_TAG, _TEXT_IN_STREAM_NAME])],
|
||||
output_streams=[
|
||||
':'.join([_CLASSIFICATIONS_TAG, _CLASSIFICATIONS_STREAM_NAME])
|
||||
],
|
||||
task_options=options)
|
||||
return cls(task_info.generate_graph_config())
|
||||
|
||||
def classify(self, text: str) -> TextClassifierResult:
|
||||
"""Performs classification on the input `text`.
|
||||
|
||||
Args:
|
||||
text: The input text.
|
||||
|
||||
Returns:
|
||||
A `TextClassifierResult` object that contains a list of text
|
||||
classifications.
|
||||
|
||||
Raises:
|
||||
ValueError: If any of the input arguments is invalid.
|
||||
RuntimeError: If text classification failed to run.
|
||||
"""
|
||||
output_packets = self._runner.process(
|
||||
{_TEXT_IN_STREAM_NAME: packet_creator.create_string(text)})
|
||||
|
||||
classification_result_proto = classifications_pb2.ClassificationResult()
|
||||
classification_result_proto.CopyFrom(
|
||||
packet_getter.get_proto(output_packets[_CLASSIFICATIONS_STREAM_NAME]))
|
||||
|
||||
return TextClassifierResult.create_from_pb2(classification_result_proto)
|
||||
@@ -0,0 +1,188 @@
|
||||
# Copyright 2022 The MediaPipe Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""MediaPipe text embedder task."""
|
||||
|
||||
import dataclasses
|
||||
from typing import Optional
|
||||
|
||||
from mediapipe.python import packet_creator
|
||||
from mediapipe.python import packet_getter
|
||||
from mediapipe.tasks.cc.components.containers.proto import embeddings_pb2
|
||||
from mediapipe.tasks.cc.components.processors.proto import embedder_options_pb2
|
||||
from mediapipe.tasks.cc.text.text_embedder.proto import text_embedder_graph_options_pb2
|
||||
from mediapipe.tasks.python.components.containers import embedding_result as embedding_result_module
|
||||
from mediapipe.tasks.python.components.utils import cosine_similarity
|
||||
from mediapipe.tasks.python.core import base_options as base_options_module
|
||||
from mediapipe.tasks.python.core import task_info as task_info_module
|
||||
from mediapipe.tasks.python.core.optional_dependencies import doc_controls
|
||||
from mediapipe.tasks.python.text.core import base_text_task_api
|
||||
|
||||
TextEmbedderResult = embedding_result_module.EmbeddingResult
|
||||
_BaseOptions = base_options_module.BaseOptions
|
||||
_TextEmbedderGraphOptionsProto = text_embedder_graph_options_pb2.TextEmbedderGraphOptions
|
||||
_EmbedderOptionsProto = embedder_options_pb2.EmbedderOptions
|
||||
_TaskInfo = task_info_module.TaskInfo
|
||||
|
||||
_EMBEDDINGS_OUT_STREAM_NAME = 'embeddings_out'
|
||||
_EMBEDDINGS_TAG = 'EMBEDDINGS'
|
||||
_TEXT_IN_STREAM_NAME = 'text_in'
|
||||
_TEXT_TAG = 'TEXT'
|
||||
_TASK_GRAPH_NAME = 'mediapipe.tasks.text.text_embedder.TextEmbedderGraph'
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TextEmbedderOptions:
|
||||
"""Options for the text embedder task.
|
||||
|
||||
Attributes:
|
||||
base_options: Base options for the text embedder task.
|
||||
l2_normalize: Whether to normalize the returned feature vector with L2 norm.
|
||||
Use this option only if the model does not already contain a native
|
||||
L2_NORMALIZATION TF Lite Op. In most cases, this is already the case and
|
||||
L2 norm is thus achieved through TF Lite inference.
|
||||
quantize: Whether the returned embedding should be quantized to bytes via
|
||||
scalar quantization. Embeddings are implicitly assumed to be unit-norm and
|
||||
therefore any dimension is guaranteed to have a value in [-1.0, 1.0]. Use
|
||||
the l2_normalize option if this is not the case.
|
||||
"""
|
||||
base_options: _BaseOptions
|
||||
l2_normalize: Optional[bool] = None
|
||||
quantize: Optional[bool] = None
|
||||
|
||||
@doc_controls.do_not_generate_docs
|
||||
def to_pb2(self) -> _TextEmbedderGraphOptionsProto:
|
||||
"""Generates an TextEmbedderOptions protobuf object."""
|
||||
base_options_proto = self.base_options.to_pb2()
|
||||
embedder_options_proto = _EmbedderOptionsProto(
|
||||
l2_normalize=self.l2_normalize, quantize=self.quantize)
|
||||
|
||||
return _TextEmbedderGraphOptionsProto(
|
||||
base_options=base_options_proto,
|
||||
embedder_options=embedder_options_proto)
|
||||
|
||||
|
||||
class TextEmbedder(base_text_task_api.BaseTextTaskApi):
|
||||
"""Class that performs embedding extraction on text.
|
||||
|
||||
This API expects a TFLite model with TFLite Model Metadata that contains the
|
||||
mandatory (described below) input tensors and output tensors. Metadata should
|
||||
contain the input process unit for the model's Tokenizer as well as input /
|
||||
output tensor metadata.
|
||||
|
||||
Input tensors:
|
||||
(kTfLiteInt32)
|
||||
- 3 input tensors of size `[batch_size x bert_max_seq_len]` with names
|
||||
"ids", "mask", and "segment_ids" representing the input ids, mask ids, and
|
||||
segment ids respectively.
|
||||
- or 1 input tensor of size `[batch_size x max_seq_len]` representing the
|
||||
input ids.
|
||||
|
||||
At least one output tensor with:
|
||||
(kTfLiteFloat32)
|
||||
- `N` components corresponding to the `N` dimensions of the returned
|
||||
feature vector for this output layer.
|
||||
- Either 2 or 4 dimensions, i.e. `[1 x N]` or `[1 x 1 x 1 x N]`.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def create_from_model_path(cls, model_path: str) -> 'TextEmbedder':
|
||||
"""Creates an `TextEmbedder` object from a TensorFlow Lite model and the default `TextEmbedderOptions`.
|
||||
|
||||
Args:
|
||||
model_path: Path to the model.
|
||||
|
||||
Returns:
|
||||
`TextEmbedder` object that's created from the model file and the default
|
||||
`TextEmbedderOptions`.
|
||||
|
||||
Raises:
|
||||
ValueError: If failed to create `TextEmbedder` object from the provided
|
||||
file such as invalid file path.
|
||||
RuntimeError: If other types of error occurred.
|
||||
"""
|
||||
base_options = _BaseOptions(model_asset_path=model_path)
|
||||
options = TextEmbedderOptions(base_options=base_options)
|
||||
return cls.create_from_options(options)
|
||||
|
||||
@classmethod
|
||||
def create_from_options(cls, options: TextEmbedderOptions) -> 'TextEmbedder':
|
||||
"""Creates the `TextEmbedder` object from text embedder options.
|
||||
|
||||
Args:
|
||||
options: Options for the text embedder task.
|
||||
|
||||
Returns:
|
||||
`TextEmbedder` object that's created from `options`.
|
||||
|
||||
Raises:
|
||||
ValueError: If failed to create `TextEmbedder` object from
|
||||
`TextEmbedderOptions` such as missing the model.
|
||||
RuntimeError: If other types of error occurred.
|
||||
"""
|
||||
task_info = _TaskInfo(
|
||||
task_graph=_TASK_GRAPH_NAME,
|
||||
input_streams=[':'.join([_TEXT_TAG, _TEXT_IN_STREAM_NAME])],
|
||||
output_streams=[
|
||||
':'.join([_EMBEDDINGS_TAG, _EMBEDDINGS_OUT_STREAM_NAME])
|
||||
],
|
||||
task_options=options)
|
||||
return cls(task_info.generate_graph_config())
|
||||
|
||||
def embed(
|
||||
self,
|
||||
text: str,
|
||||
) -> TextEmbedderResult:
|
||||
"""Performs text embedding extraction on the provided text.
|
||||
|
||||
Args:
|
||||
text: The input text.
|
||||
|
||||
Returns:
|
||||
An embedding result object that contains a list of embeddings.
|
||||
|
||||
Raises:
|
||||
ValueError: If any of the input arguments is invalid.
|
||||
RuntimeError: If text embedder failed to run.
|
||||
"""
|
||||
output_packets = self._runner.process(
|
||||
{_TEXT_IN_STREAM_NAME: packet_creator.create_string(text)})
|
||||
|
||||
embedding_result_proto = embeddings_pb2.EmbeddingResult()
|
||||
embedding_result_proto.CopyFrom(
|
||||
packet_getter.get_proto(output_packets[_EMBEDDINGS_OUT_STREAM_NAME]))
|
||||
|
||||
return TextEmbedderResult.create_from_pb2(embedding_result_proto)
|
||||
|
||||
@classmethod
|
||||
def cosine_similarity(cls, u: embedding_result_module.Embedding,
|
||||
v: embedding_result_module.Embedding) -> float:
|
||||
"""Utility function to compute cosine similarity between two embedding entries.
|
||||
|
||||
May return an InvalidArgumentError if e.g. the feature vectors are
|
||||
of different types (quantized vs. float), have different sizes, or have a
|
||||
an L2-norm of 0.
|
||||
|
||||
Args:
|
||||
u: An embedding entry.
|
||||
v: An embedding entry.
|
||||
|
||||
Returns:
|
||||
The cosine similarity for the two embeddings.
|
||||
|
||||
Raises:
|
||||
ValueError: May return an error if e.g. the feature vectors are of
|
||||
different types (quantized vs. float), have different sizes, or have
|
||||
an L2-norm of 0.
|
||||
"""
|
||||
return cosine_similarity.cosine_similarity(u, v)
|
||||
Reference in New Issue
Block a user