Skip to content

Commit

Permalink
Merge pull request #759 from guardrails-ai/0.4.4-dev
Browse files Browse the repository at this point in the history 
0.4.4 dev
  • Loading branch information
@zsimjee
zsimjee authored May 22, 2024
2 parents 4fb20f7 + 7dc0819 commit 5363c07
Show file tree
Hide file tree
Showing 25 changed files with 4,405 additions and 3,330 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
pydantic-version: ["1.10.9", "2.4.2"]
openai-version: ["1.2.4"]
openai-version: ["1.30.1"]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
Expand Down Expand Up @@ -87,7 +87,7 @@ jobs:
# dependencies: ['dev', 'full']
dependencies: ["full"]
pydantic-version: ["1.10.9", "2.4.2"]
openai-version: ["1.2.4"]
openai-version: ["1.30.1"]
steps:
- uses: actions/checkout@v4

Expand Down
205 changes: 83 additions & 122 deletions docs/examples/bug_free_python_code.ipynb
View file

Large diffs are not rendered by default.

202 changes: 22 additions & 180 deletions docs/examples/check_for_pii.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,149 +14,61 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
"/home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/torch/cuda/__init__.py:611: UserWarning: Can't initialize NVML\n",
" warnings.warn(\"Can't initialize NVML\")\n",
"\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
"\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
"You can now load the package via spacy.load('en_core_web_lg')\n"
"You can now load the package via spacy.load('en_core_web_lg')\n",
"Installing hub:\u001b[35m/\u001b[0m\u001b[35m/guardrails/\u001b[0m\u001b[95mdetect_pii...\u001b[0m\n",
"✅Successfully installed guardrails/detect_pii!\n",
"\n",
"\n"
]
}
],
"source": [
"# Install the necessary packages\n",
"! pip install presidio-analyzer presidio-anonymizer -q\n",
"! python -m spacy download en_core_web_lg -q"
"! python -m spacy download en_core_web_lg -q\n",
"\n",
"! guardrails hub install hub://guardrails/detect_pii --quiet"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/torch/cuda/__init__.py:611: UserWarning: Can't initialize NVML\n",
" warnings.warn(\"Can't initialize NVML\")\n"
]
}
],
"outputs": [],
"source": [
"# Import the guardrails package\n",
"from guardrails.hub import DetectPII\n",
"import guardrails as gd\n",
"from guardrails.validators import PIIFilter\n",
"from rich import print"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"nlp_engine not provided, creating default.\n",
"configuration file /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/conf/default.yaml not found. Using default config: {'nlp_engine_name': 'spacy', 'models': [{'lang_code': 'en', 'model_name': 'en_core_web_lg'}]}.\n",
"configuration file is missing 'ner_model_configuration'. Using default\n",
"model_to_presidio_entity_mapping is missing from configuration, using default\n",
"low_score_entity_names is missing from configuration, using default\n",
"labels_to_ignore is missing from configuration, using default\n",
"Created NLP engine: spacy. Loaded models: ['en']\n",
"registry not provided, creating default.\n",
"Loaded recognizer: UsBankRecognizer\n",
"Loaded recognizer: UsLicenseRecognizer\n",
"Loaded recognizer: UsItinRecognizer\n",
"Loaded recognizer: UsPassportRecognizer\n",
"Loaded recognizer: UsSsnRecognizer\n",
"Loaded recognizer: NhsRecognizer\n",
"Loaded recognizer: SgFinRecognizer\n",
"Loaded recognizer: AuAbnRecognizer\n",
"Loaded recognizer: AuAcnRecognizer\n",
"Loaded recognizer: AuTfnRecognizer\n",
"Loaded recognizer: AuMedicareRecognizer\n",
"Loaded recognizer: InPanRecognizer\n",
"Loaded recognizer: CreditCardRecognizer\n",
"Loaded recognizer: CryptoRecognizer\n",
"Loaded recognizer: DateRecognizer\n",
"Loaded recognizer: EmailRecognizer\n",
"Loaded recognizer: IbanRecognizer\n",
"Loaded recognizer: IpRecognizer\n",
"Loaded recognizer: MedicalLicenseRecognizer\n",
"Loaded recognizer: PhoneRecognizer\n",
"Loaded recognizer: UrlRecognizer\n",
"Loaded recognizer: SpacyRecognizer\n",
"nlp_engine not provided, creating default.\n",
"configuration file /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/conf/default.yaml not found. Using default config: {'nlp_engine_name': 'spacy', 'models': [{'lang_code': 'en', 'model_name': 'en_core_web_lg'}]}.\n",
"configuration file is missing 'ner_model_configuration'. Using default\n",
"model_to_presidio_entity_mapping is missing from configuration, using default\n",
"low_score_entity_names is missing from configuration, using default\n",
"labels_to_ignore is missing from configuration, using default\n",
"Created NLP engine: spacy. Loaded models: ['en']\n",
"registry not provided, creating default.\n",
"Loaded recognizer: UsBankRecognizer\n",
"Loaded recognizer: UsLicenseRecognizer\n",
"Loaded recognizer: UsItinRecognizer\n",
"Loaded recognizer: UsPassportRecognizer\n",
"Loaded recognizer: UsSsnRecognizer\n",
"Loaded recognizer: NhsRecognizer\n",
"Loaded recognizer: SgFinRecognizer\n",
"Loaded recognizer: AuAbnRecognizer\n",
"Loaded recognizer: AuAcnRecognizer\n",
"Loaded recognizer: AuTfnRecognizer\n",
"Loaded recognizer: AuMedicareRecognizer\n",
"Loaded recognizer: InPanRecognizer\n",
"Loaded recognizer: CreditCardRecognizer\n",
"Loaded recognizer: CryptoRecognizer\n",
"Loaded recognizer: DateRecognizer\n",
"Loaded recognizer: EmailRecognizer\n",
"Loaded recognizer: IbanRecognizer\n",
"Loaded recognizer: IpRecognizer\n",
"Loaded recognizer: MedicalLicenseRecognizer\n",
"Loaded recognizer: PhoneRecognizer\n",
"Loaded recognizer: UrlRecognizer\n",
"Loaded recognizer: SpacyRecognizer\n"
]
}
],
"outputs": [],
"source": [
"# Create Guard object with this validator\n",
"# One can specify either pre-defined set of PII or SPI (Sensitive Personal Information) entities by passing in the `pii` or `spi` argument respectively.\n",
"# It can be passed either durring intialization or later through the metadata argument in parse method.\n",
"\n",
"# One can also pass in a list of entities supported by Presidio to the `pii_entities` argument.\n",
"guard = gd.Guard.from_string(\n",
" validators=[PIIFilter(pii_entities=\"pii\", on_fail=\"fix\")],\n",
" description=\"testmeout\",\n",
")"
"guard = gd.Guard().use(DetectPII(pii_entities=\"pii\", on_fail=\"fix\"))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Entity DOMAIN_NAME doesn't have the corresponding recognizer in language : en\n"
]
},
{
"data": {
"text/html": [
Expand Down Expand Up @@ -203,7 +115,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 14,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -253,88 +165,18 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"nlp_engine not provided, creating default.\n",
"configuration file /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/conf/default.yaml not found. Using default config: {'nlp_engine_name': 'spacy', 'models': [{'lang_code': 'en', 'model_name': 'en_core_web_lg'}]}.\n",
"configuration file is missing 'ner_model_configuration'. Using default\n",
"model_to_presidio_entity_mapping is missing from configuration, using default\n",
"low_score_entity_names is missing from configuration, using default\n",
"labels_to_ignore is missing from configuration, using default\n",
"Created NLP engine: spacy. Loaded models: ['en']\n",
"registry not provided, creating default.\n",
"Loaded recognizer: UsBankRecognizer\n",
"Loaded recognizer: UsLicenseRecognizer\n",
"Loaded recognizer: UsItinRecognizer\n",
"Loaded recognizer: UsPassportRecognizer\n",
"Loaded recognizer: UsSsnRecognizer\n",
"Loaded recognizer: NhsRecognizer\n",
"Loaded recognizer: SgFinRecognizer\n",
"Loaded recognizer: AuAbnRecognizer\n",
"Loaded recognizer: AuAcnRecognizer\n",
"Loaded recognizer: AuTfnRecognizer\n",
"Loaded recognizer: AuMedicareRecognizer\n",
"Loaded recognizer: InPanRecognizer\n",
"Loaded recognizer: CreditCardRecognizer\n",
"Loaded recognizer: CryptoRecognizer\n",
"Loaded recognizer: DateRecognizer\n",
"Loaded recognizer: EmailRecognizer\n",
"Loaded recognizer: IbanRecognizer\n",
"Loaded recognizer: IpRecognizer\n",
"Loaded recognizer: MedicalLicenseRecognizer\n",
"Loaded recognizer: PhoneRecognizer\n",
"Loaded recognizer: UrlRecognizer\n",
"Loaded recognizer: SpacyRecognizer\n",
"nlp_engine not provided, creating default.\n",
"configuration file /home/zayd/workspace/guardrails/.venv/lib/python3.9/site-packages/conf/default.yaml not found. Using default config: {'nlp_engine_name': 'spacy', 'models': [{'lang_code': 'en', 'model_name': 'en_core_web_lg'}]}.\n",
"configuration file is missing 'ner_model_configuration'. Using default\n",
"model_to_presidio_entity_mapping is missing from configuration, using default\n",
"low_score_entity_names is missing from configuration, using default\n",
"labels_to_ignore is missing from configuration, using default\n",
"Created NLP engine: spacy. Loaded models: ['en']\n",
"registry not provided, creating default.\n",
"Loaded recognizer: UsBankRecognizer\n",
"Loaded recognizer: UsLicenseRecognizer\n",
"Loaded recognizer: UsItinRecognizer\n",
"Loaded recognizer: UsPassportRecognizer\n",
"Loaded recognizer: UsSsnRecognizer\n",
"Loaded recognizer: NhsRecognizer\n",
"Loaded recognizer: SgFinRecognizer\n",
"Loaded recognizer: AuAbnRecognizer\n",
"Loaded recognizer: AuAcnRecognizer\n",
"Loaded recognizer: AuTfnRecognizer\n",
"Loaded recognizer: AuMedicareRecognizer\n",
"Loaded recognizer: InPanRecognizer\n",
"Loaded recognizer: CreditCardRecognizer\n",
"Loaded recognizer: CryptoRecognizer\n",
"Loaded recognizer: DateRecognizer\n",
"Loaded recognizer: EmailRecognizer\n",
"Loaded recognizer: IbanRecognizer\n",
"Loaded recognizer: IpRecognizer\n",
"Loaded recognizer: MedicalLicenseRecognizer\n",
"Loaded recognizer: PhoneRecognizer\n",
"Loaded recognizer: UrlRecognizer\n",
"Loaded recognizer: SpacyRecognizer\n"
]
}
],
"outputs": [],
"source": [
"# Let's try with SPI entities\n",
"# Create a new guard object\n",
"guard = gd.Guard.from_string(\n",
" validators=[PIIFilter(pii_entities=\"spi\", on_fail=\"fix\")],\n",
" description=\"testmeout\",\n",
")"
"guard = gd.Guard().use(DetectPII(pii_entities=\"spi\", on_fail=\"fix\"))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 16,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -384,7 +226,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 17,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -450,7 +292,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.17"
"version": "3.10.0"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit 5363c07

Please sign in to comment.