Commit aae2a364 authored by ozzeh's avatar ozzeh
Browse files

Initial checkin

parents
schema_registry.egg-info/
\ No newline at end of file
[[package]]
category = "dev"
description = "Atomic file writes."
marker = "sys_platform == \"win32\""
name = "atomicwrites"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "1.4.0"
[[package]]
category = "dev"
description = "Classes Without Boilerplate"
name = "attrs"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "20.3.0"
[package.extras]
dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"]
docs = ["furo", "sphinx", "zope.interface"]
tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
tests_no_zope = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"]
[[package]]
category = "dev"
description = "Cross-platform colored terminal text."
marker = "sys_platform == \"win32\""
name = "colorama"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.4.4"
[[package]]
category = "dev"
description = "More routines for operating on iterables, beyond itertools"
name = "more-itertools"
optional = false
python-versions = ">=3.5"
version = "8.6.0"
[[package]]
category = "dev"
description = "Core utilities for Python packages"
name = "packaging"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "20.4"
[package.dependencies]
pyparsing = ">=2.0.2"
six = "*"
[[package]]
category = "dev"
description = "plugin and hook calling mechanisms for python"
name = "pluggy"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.13.1"
[package.extras]
dev = ["pre-commit", "tox"]
[[package]]
category = "dev"
description = "library with cross-python path, ini-parsing, io, code, log facilities"
name = "py"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "1.9.0"
[[package]]
category = "dev"
description = "Python parsing module"
name = "pyparsing"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
version = "2.4.7"
[[package]]
category = "dev"
description = "pytest: simple powerful testing with Python"
name = "pytest"
optional = false
python-versions = ">=3.5"
version = "5.4.3"
[package.dependencies]
atomicwrites = ">=1.0"
attrs = ">=17.4.0"
colorama = "*"
more-itertools = ">=4.0.0"
packaging = "*"
pluggy = ">=0.12,<1.0"
py = ">=1.5.0"
wcwidth = "*"
[package.extras]
checkqa-mypy = ["mypy (v0.761)"]
testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
[[package]]
category = "dev"
description = "Python 2 and 3 compatibility utilities"
name = "six"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
version = "1.15.0"
[[package]]
category = "dev"
description = "Measures the displayed width of unicode strings in a terminal"
name = "wcwidth"
optional = false
python-versions = "*"
version = "0.2.5"
[metadata]
content-hash = "c27944f25b55067b06883f1cea204be7d97841a4b8228fab69b91895347494ad"
lock-version = "1.0"
python-versions = "^3.8"
[metadata.files]
atomicwrites = [
{file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"},
{file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
]
attrs = [
{file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"},
{file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"},
]
colorama = [
{file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
{file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},
]
more-itertools = [
{file = "more-itertools-8.6.0.tar.gz", hash = "sha256:b3a9005928e5bed54076e6e549c792b306fddfe72b2d1d22dd63d42d5d3899cf"},
{file = "more_itertools-8.6.0-py3-none-any.whl", hash = "sha256:8e1a2a43b2f2727425f2b5839587ae37093f19153dc26c0927d1048ff6557330"},
]
packaging = [
{file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
{file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"},
]
pluggy = [
{file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
{file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
]
py = [
{file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"},
{file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"},
]
pyparsing = [
{file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
{file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"},
]
pytest = [
{file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
{file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
]
six = [
{file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
{file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
]
wcwidth = [
{file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
{file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
]
[tool.poetry]
name = "schema-registry"
version = "0.1.0"
description = ""
authors = ["ozzeh <ozzeh@pleaseignore.com>"]
[tool.poetry.dependencies]
python = "^3.8"
[tool.poetry.dev-dependencies]
pytest = "^5.2"
[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"
from .client import SchemaRegistry
from .errors import SchemaRegistryError, ModelNotRegisteredError
\ No newline at end of file
import json
import sys
import logging
from typing import List, Optional, Dict, Type
from datetime import datetime
import boto3
from pydantic import BaseModel
from .models import (
_SchemaPageModel,
_SchemaVersionsPageModel,
_SchemaVersionModel,
_SchemaContentModel,
_SchemaCreateUpdateModel,
)
from .errors import SchemaRegistryError, ModelNotRegisteredError
logger = logging.getLogger("schema_registry")
class Schema:
def __init__(self, client, registry_name, schema_name):
self.schema_client = client
self.registry_name: str = registry_name
self.schema_name = schema_name
self._versions: Dict[str, _SchemaContentModel] = {}
self.default_version: int = 0
self._load_versions()
self.default_version = sorted(self._versions.items(), reverse=True)[0][0]
def _load_versions(self):
paginator = self.schema_client.get_paginator("list_schema_versions")
page_options = dict(
RegistryName=self.registry_name, SchemaName=self.schema_name
)
for raw_page in paginator.paginate(**page_options):
schema_versions: _SchemaVersionsPageModel = (
_SchemaVersionsPageModel.parse_obj(raw_page)
)
for version in schema_versions.schema_versions:
self._versions[
version.schema_version
] = self._get_schema_version_content(version)
def _get_schema_version_content(self, schema_version: _SchemaVersionModel) -> _SchemaContentModel:
describe_opts = dict(
RegistryName=self.registry_name,
SchemaName=self.schema_name,
SchemaVersion=schema_version.schema_version,
)
response = self.schema_client.describe_schema(**describe_opts)
content: _SchemaContentModel = _SchemaContentModel.parse_obj(response)
return content
def get(self, version=None):
if not version:
return self._versions[self.default_version]
else:
return self._versions[version]
def __repr__(self):
return f"Schema<{self.schema_name}, versions: {len(self._versions)}, default version: {self.default_version}>"
class SchemaRegistry:
def __init__(
self, registry_name: Optional[str] = None, *, prefix: str = None, **boto_opts
):
self.registry_name: str = registry_name or "discovered-schemas"
self.session = boto3.Session(**boto_opts)
self.schema_client = self.session.client("schemas")
self.prefix = prefix
self._schemas: Dict[str, Schema] = {}
self._model_schemas: Dict[Type[BaseModel], _SchemaCreateUpdateModel] = {}
self._load_schemas()
def _load_schemas(self):
paginator = self.schema_client.get_paginator("list_schemas")
page_options = dict(RegistryName=self.registry_name)
if self.prefix:
page_options["SchemaNamePrefix"] = self.prefix
for raw_page in paginator.paginate(**page_options):
schema_page: _SchemaPageModel = _SchemaPageModel.parse_obj(raw_page)
for schema in schema_page.schemas:
self._schemas[schema.schema_name] = Schema(
self.schema_client, self.registry_name, schema.schema_name
)
def register_model(
self, sender: str, model: Type[BaseModel]
) -> _SchemaCreateUpdateModel:
schema_name = "{}@{}".format(sender, model.__name__)
opts = dict(
Content=model.schema_json(),
RegistryName=self.registry_name,
SchemaName=schema_name,
Type="JSONSchemaDraft4",
)
description = model.schema().get("description")
if description:
opts["Description"] = description
if schema_name not in self._schemas:
response = self.schema_client.create_schema(**opts)
schema_info = _SchemaCreateUpdateModel.parse_obj(response)
self._model_schemas[model] = schema_info
return schema_info
else:
try:
response = self.schema_client.update_schema(**opts)
except self.schema_client.exceptions.ConflictException:
logger.info("Schema did not change so we don't have to worry")
schema_info = self._schemas[schema_name].get()
self._model_schemas[model] = schema_info
return schema_info
else:
schema_info = _SchemaCreateUpdateModel.parse_obj(response)
self._model_schemas[model] = schema_info
return schema_info
def schema_for_model(self, model: Type[BaseModel]) -> dict:
if model not in self._model_schemas:
raise ModelNotRegisteredError(model)
return self._model_schemas[model].dict(include=set(["schema_arn", "schema_name", "schema_version"]))
class SchemaRegistryError(Exception):
pass
class ModelNotRegisteredError(SchemaRegistryError):
def __init__(self, model):
self.model = model
\ No newline at end of file
from typing import List, Optional, Dict, Literal
from datetime import datetime
from pydantic import create_model, BaseModel, Field, PrivateAttr
from .utils import camel_generator
import json
class _AWSResponseModel(BaseModel):
class Config:
alias_generator = camel_generator
class _SchemaCreateUpdateModel(_AWSResponseModel):
description: Optional[str]
last_modified: datetime
schema_arn: str
schema_name: str
schema_version: str
tags: Dict[str, str]
type_: Literal["OpenApi3", "JSONSchemaDraft4"] = Field(..., alias="Type")
version_created_date: datetime
class _SchemaContentModel(_SchemaCreateUpdateModel):
content: str
_content: dict = PrivateAttr({})
def content_dict(self):
if not self._content:
self._content = json.loads(self.content)
return self._content
class _SchemaVersionModel(_AWSResponseModel):
schema_arn: str
schema_name: str
schema_version: str
type_: Literal["OpenApi3", "JSONSchemaDraft4"] = Field(..., alias="Type")
class _SchemaModel(_AWSResponseModel):
last_modified: datetime
schema_arn: str
schema_name: str
tags: Dict[str, str]
version_count: int
class _SchemaVersionsPageModel(_AWSResponseModel):
schema_versions: List[_SchemaVersionModel]
class _SchemaPageModel(_AWSResponseModel):
schemas: List[_SchemaModel]
def camel_generator(string: str) -> str:
return "".join(word.capitalize() for word in string.split("_"))
from typing import Optional
import pytest
import boto3
from typing import List
from schema_registry import SchemaRegistry
from pydantic import BaseModel
import logging
logging.basicConfig()
log = logging.getLogger("schema_registry")
log.setLevel(logging.DEBUG)
log.propagate = True
@pytest.fixture(scope="module")
def named_registry():
_registry = SchemaRegistry(registry_name="TAPI-TEST")
yield _registry
@pytest.fixture(scope="module")
def dynamic_registry():
_registry = SchemaRegistry()
yield _registry
@pytest.fixture(scope="module")
def test_model():
class TestingModel(BaseModel):
name: str
description: Optional[str]
yield TestingModel
@pytest.fixture(scope="module")
def complex_model():
class Group(BaseModel):
id: int
name: str
class ComplexModel(BaseModel):
"""Hi mom"""
name: str
description: Optional[str]
groups: List[Group]
yield ComplexModel
def test_load_named_schemas(test_model, named_registry):
pass
def test_schema_registration(test_model, named_registry):
schema_info = named_registry.register_model("com.pleaseignore.tvm.test", test_model)
def test_registered_model(test_model, named_registry):
model_info = named_registry.schema_for_model(test_model)
def test_complex_mode(complex_model, named_registry):
named_registry.register_model("com.pleaseignore.tvm.test", complex_model)
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment