Skip to content

Commit

Permalink
Add time format enum
Browse files Browse the repository at this point in the history
  • Loading branch information
marcsiftstack committed Oct 9, 2024
1 parent 0829281 commit f732661
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 43 deletions.
18 changes: 13 additions & 5 deletions python/lib/sift_py/data_import/_status_test.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import json

import pytest
from pytest_mock import MockFixture
from sift_py.data_import.status import DataImportStatus, DataImportStatusValue
from sift_py.data_import.status import DataImportStatus, DataImportStatusType
from sift_py.rest import SiftRestConfig

rest_config: SiftRestConfig = {
Expand Down Expand Up @@ -32,26 +33,33 @@ def test_get_status(mocker: MockFixture):
status_code=200, text=json.dumps({"dataImport": {"status": "DATA_IMPORT_STATUS_SUCCEEDED"}})
)
status = DataImportStatus(rest_config, "123-123-123")
assert status.get_status() == DataImportStatusValue.SUCCEEDED
assert status.get_status() == DataImportStatusType.SUCCEEDED

mock_requests_post.return_value = MockResponse(
status_code=200, text=json.dumps({"dataImport": {"status": "DATA_IMPORT_STATUS_PENDING"}})
)
status = DataImportStatus(rest_config, "123-123-123")
assert status.get_status() == DataImportStatusValue.PENDING
assert status.get_status() == DataImportStatusType.PENDING

mock_requests_post.return_value = MockResponse(
status_code=200,
text=json.dumps({"dataImport": {"status": "DATA_IMPORT_STATUS_IN_PROGRESS"}}),
)
status = DataImportStatus(rest_config, "123-123-123")
assert status.get_status() == DataImportStatusValue.IN_PROGRESS
assert status.get_status() == DataImportStatusType.IN_PROGRESS

mock_requests_post.return_value = MockResponse(
status_code=200, text=json.dumps({"dataImport": {"status": "DATA_IMPORT_STATUS_FAILED"}})
)
status = DataImportStatus(rest_config, "123-123-123")
assert status.get_status() == DataImportStatusValue.FAILED
assert status.get_status() == DataImportStatusType.FAILED

with pytest.raises(Exception, match="Unknown data import status"):
mock_requests_post.return_value = MockResponse(
status_code=200, text=json.dumps({"dataImport": {"status": "INVALID_STATUS"}})
)
status = DataImportStatus(rest_config, "123-123-123")
status.get_status()


def test_wait_success(mocker: MockFixture):
Expand Down
34 changes: 7 additions & 27 deletions python/lib/sift_py/data_import/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,9 @@

from pydantic import BaseModel, ConfigDict, model_validator
from pydantic_core import PydanticCustomError
from sift_py.data_import.time_format import TimeFormatType
from sift_py.ingestion.channel import ChannelBitFieldElement, ChannelDataType, ChannelEnumType

VALID_TIME_FORMATS = [
"TIME_FORMAT_ABSOLUTE_RFC3339",
"TIME_FORMAT_ABSOLUTE_DATETIME",
"TIME_FORMAT_ABSOLUTE_UNIX_SECONDS",
"TIME_FORMAT_ABSOLUTE_UNIX_MILLISECONDS",
"TIME_FORMAT_ABSOLUTE_UNIX_MICROSECONDS",
"TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS",
"TIME_FORMAT_RELATIVE_NANOSECONDS",
"TIME_FORMAT_RELATIVE_MICROSECONDS",
"TIME_FORMAT_RELATIVE_MILLISECONDS",
"TIME_FORMAT_RELATIVE_SECONDS",
"TIME_FORMAT_RELATIVE_MINUTES",
"TIME_FORMAT_RELATIVE_HOURS",
]


class CsvConfig:
def __init__(self, config_info) -> None:
Expand Down Expand Up @@ -68,23 +54,18 @@ class _TimeColumn(_BaseModel):
relative_start_time: Optional[str] = None

@model_validator(mode="after")
def validate_format(self):
if self.format not in VALID_TIME_FORMATS:
def validate_time(self):
format = TimeFormatType.from_str(self.format)
if format is None:
raise PydanticCustomError(
"invalid_config_error",
"Invalid time format: {format}.\nValid options: {valid}",
{"format": self.format, "valid": ", ".join(VALID_TIME_FORMATS)},
"invalid_config_error", f"Invalid time format: {self.format}."
)

return self

@model_validator(mode="after")
def validate_relative_time(self):
if self.format.startswith("TIME_FORMAT_RELATIVE_"):
if format.is_relative():
if self.relative_start_time is None:
raise PydanticCustomError("invalid_config_error", "Missing 'relative_start_time'")
else:
if self.relative_start_time:
if self.relative_start_time is not None:
raise PydanticCustomError(
"invalid_config_error",
"'relative_start_time' specified for non relative time format.",
Expand All @@ -108,7 +89,6 @@ def validate_data_type(self):
raise PydanticCustomError(
"invalid_config_error", f"Invalid data_type: {self.data_type}."
)

return self

@model_validator(mode="after")
Expand Down
26 changes: 15 additions & 11 deletions python/lib/sift_py/data_import/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from sift_py.rest import SiftRestConfig, compute_uri


class DataImportStatusValue(Enum):
class DataImportStatusType(Enum):
SUCCEEDED = "DATA_IMPORT_STATUS_SUCCEEDED"
PENDING = "DATA_IMPORT_STATUS_PENDING"
IN_PROGRESS = "DATA_IMPORT_STATUS_IN_PROGRESS"
Expand All @@ -22,8 +22,8 @@ def from_str(cls, val: str):
return cls.IN_PROGRESS
elif val == cls.FAILED.value:
return cls.FAILED
else:
raise ValueError("Argument 'val' is not a valid status.")

return None


class DataImportStatus:
Expand All @@ -36,27 +36,31 @@ def __init__(self, restconf: SiftRestConfig, data_import_id: str):
self._status_uri = urljoin(base_uri, self.STATUS_PATH)
self._apikey = restconf["apikey"]

def get_status(self) -> DataImportStatusValue:
def get_status(self) -> DataImportStatusType:
response = requests.get(
url=f"{self._status_uri}/{self._data_import_id}",
headers={"Authorization": f"Bearer {self._apikey}"},
)
response.raise_for_status()

status = response.json().get("dataImport").get("status")
return DataImportStatusValue.from_str(status)
status_text = response.json().get("dataImport").get("status")
status = DataImportStatusType.from_str(status_text)
if status is None:
raise Exception(f"Unknown data import status: {status_text}")

return status

def wait_until_complete(self) -> bool:
polling_interval = 1
while True:
status: DataImportStatusValue = self.get_status()
if status == DataImportStatusValue.SUCCEEDED:
status: DataImportStatusType = self.get_status()
if status == DataImportStatusType.SUCCEEDED:
return True
elif status == DataImportStatusValue.PENDING:
elif status == DataImportStatusType.PENDING:
pass
elif status == DataImportStatusValue.IN_PROGRESS:
elif status == DataImportStatusType.IN_PROGRESS:
pass
elif status == DataImportStatusValue.FAILED:
elif status == DataImportStatusType.FAILED:
return False
else:
raise Exception(f"Unknown status: {status}")
Expand Down
55 changes: 55 additions & 0 deletions python/lib/sift_py/data_import/time_format.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
from enum import Enum


class TimeFormatType(Enum):
ABSOLUTE_RFC339 = "TIME_FORMAT_ABSOLUTE_RFC3339"
ABSOLUTE_DATETIME = "TIME_FORMAT_ABSOLUTE_DATETIME"
ABSOLUTE_UNIX_SECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_SECONDS"
ABSOLUTE_UNIX_MILLISECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_MILLISECONDS"
ABSOLUTE_UNIX_MICROSECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_MICROSECONDS"
ABSOLUTE_UNIX_NANOSECONDS = "TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS"
RELATIVE_NANOSECONDS = "TIME_FORMAT_RELATIVE_NANOSECONDS"
RELATIVE_MICROSECONDS = "TIME_FORMAT_RELATIVE_MICROSECONDS"
RELATIVE_MILLISECONDS = "TIME_FORMAT_RELATIVE_MILLISECONDS"
RELATIVE_SECONDS = "TIME_FORMAT_RELATIVE_SECONDS"
RELATIVE_MINUTES = "TIME_FORMAT_RELATIVE_MINUTES"
RELATIVE_HOURS = "TIME_FORMAT_RELATIVE_HOURS"

@classmethod
def from_str(cls, val: str):
if val == cls.ABSOLUTE_RFC339.value:
return cls.ABSOLUTE_RFC339
elif val == cls.ABSOLUTE_DATETIME.value:
return cls.ABSOLUTE_DATETIME
elif val == cls.ABSOLUTE_UNIX_SECONDS.value:
return cls.ABSOLUTE_UNIX_SECONDS
elif val == cls.ABSOLUTE_UNIX_MILLISECONDS.value:
return cls.ABSOLUTE_UNIX_MILLISECONDS
elif val == cls.ABSOLUTE_UNIX_MICROSECONDS.value:
return cls.ABSOLUTE_UNIX_MICROSECONDS
elif val == cls.ABSOLUTE_UNIX_NANOSECONDS.value:
return cls.ABSOLUTE_UNIX_NANOSECONDS
elif val == cls.RELATIVE_NANOSECONDS.value:
return cls.RELATIVE_NANOSECONDS
elif val == cls.RELATIVE_MICROSECONDS.value:
return cls.RELATIVE_MICROSECONDS
elif val == cls.RELATIVE_MILLISECONDS.value:
return cls.RELATIVE_MILLISECONDS
elif val == cls.RELATIVE_SECONDS.value:
return cls.RELATIVE_SECONDS
elif val == cls.RELATIVE_MINUTES.value:
return cls.RELATIVE_MINUTES
elif val == cls.RELATIVE_HOURS.value:
return cls.RELATIVE_HOURS

return None

def is_relative(self):
return self in [
self.RELATIVE_NANOSECONDS,
self.RELATIVE_MICROSECONDS,
self.RELATIVE_MILLISECONDS,
self.RELATIVE_SECONDS,
self.RELATIVE_MINUTES,
self.RELATIVE_HOURS,
]

0 comments on commit f732661

Please sign in to comment.