From de3757d922f62ee6afb1abd95c8f4953df0ff6e5 Mon Sep 17 00:00:00 2001
From: Titusz Pan <titusz.pan@gmail.com>
Date: Fri, 15 Dec 2023 13:40:55 +0100
Subject: [PATCH] Deployed 1e9a785 with MkDocs version: 1.5.3

---
 changelog/index.html       |  12 +++++++++
 conformance/index.html     |   2 +-
 options/options/index.html |  36 ++++++++++++-------------
 search/search_index.json   |   2 +-
 sitemap.xml                |  54 ++++++++++++++++++-------------------
 sitemap.xml.gz             | Bin 378 -> 379 bytes
 6 files changed, 59 insertions(+), 47 deletions(-)

diff --git a/changelog/index.html b/changelog/index.html
index e07f707..00115fa 100755
--- a/changelog/index.html
+++ b/changelog/index.html
@@ -1114,6 +1114,13 @@
     </label>
     <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
       
+        <li class="md-nav__item">
+  <a href="#106-2023-12-15" class="md-nav__link">
+    [1.0.6] - 2023-12-15
+  </a>
+  
+</li>
+      
         <li class="md-nav__item">
   <a href="#105-2023-12-07" class="md-nav__link">
     [1.0.5] - 2023-12-07
@@ -1351,6 +1358,11 @@
 
 
 <h1 id="changelog">Changelog<a class="headerlink" href="#changelog" title="Permanent link">#</a></h1>
+<h2 id="106-2023-12-15">[1.0.6] - 2023-12-15<a class="headerlink" href="#106-2023-12-15" title="Permanent link">#</a></h2>
+<ul>
+<li>Added wheel package testing to CI</li>
+<li>Fixed incompatible dependencies</li>
+</ul>
 <h2 id="105-2023-12-07">[1.0.5] - 2023-12-07<a class="headerlink" href="#105-2023-12-07" title="Permanent link">#</a></h2>
 <ul>
 <li>Improved simhash performance</li>
diff --git a/conformance/index.html b/conformance/index.html
index 499b90e..bd0864b 100755
--- a/conformance/index.html
+++ b/conformance/index.html
@@ -1167,7 +1167,7 @@ <h1 id="iscc-conformance-testing">ISCC - Conformance Testing<a class="headerlink
   
       <p>An application that claims ISCC conformance MUST pass all core functions from the
 ISCC conformance test suite. The test suite is available as JSON data on
-<a href="https://raw.githubusercontent.com/iscc/iscc-core/master/iscc_core/data.json">Github</a>.
+<a href="https://raw.githubusercontent.com/iscc/iscc-core/master/iscc_core/data.json">GitHub</a>.
 Test data is structured as follows:</p>
 <div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="p">{</span>
 <a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="w">  </span><span class="nt">&quot;&lt;function_name&gt;&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
diff --git a/options/options/index.html b/options/options/index.html
index fe1a2e1..7864d43 100755
--- a/options/options/index.html
+++ b/options/options/index.html
@@ -1400,8 +1400,8 @@ <h4 id="iscc_core.options.CoreOptions.meta_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.meta_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1430,8 +1430,8 @@ <h4 id="iscc_core.options.CoreOptions.meta_trim_name" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.meta_trim_name" class="headerlink" title="Permanent link">#</a></h4>
@@ -1459,8 +1459,8 @@ <h4 id="iscc_core.options.CoreOptions.meta_trim_description" class="doc doc-head
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.meta_trim_description" class="headerlink" title="Permanent link">#</a></h4>
@@ -1489,8 +1489,8 @@ <h4 id="iscc_core.options.CoreOptions.meta_ngram_size_text" class="doc doc-headi
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.meta_ngram_size_text" class="headerlink" title="Permanent link">#</a></h4>
@@ -1519,8 +1519,8 @@ <h4 id="iscc_core.options.CoreOptions.meta_ngram_size_bytes" class="doc doc-head
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.meta_ngram_size_bytes" class="headerlink" title="Permanent link">#</a></h4>
@@ -1549,8 +1549,8 @@ <h4 id="iscc_core.options.CoreOptions.text_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.text_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1579,8 +1579,8 @@ <h4 id="iscc_core.options.CoreOptions.text_ngram_size" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.text_ngram_size" class="headerlink" title="Permanent link">#</a></h4>
@@ -1609,8 +1609,8 @@ <h4 id="iscc_core.options.CoreOptions.text_unicode_filter" class="doc doc-headin
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.text_unicode_filter" class="headerlink" title="Permanent link">#</a></h4>
@@ -1639,8 +1639,8 @@ <h4 id="iscc_core.options.CoreOptions.text_newlines" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.text_newlines" class="headerlink" title="Permanent link">#</a></h4>
@@ -1679,8 +1679,8 @@ <h4 id="iscc_core.options.CoreOptions.image_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.image_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1709,8 +1709,8 @@ <h4 id="iscc_core.options.CoreOptions.audio_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.audio_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1739,8 +1739,8 @@ <h4 id="iscc_core.options.CoreOptions.video_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.video_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1769,8 +1769,8 @@ <h4 id="iscc_core.options.CoreOptions.data_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.data_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1799,8 +1799,8 @@ <h4 id="iscc_core.options.CoreOptions.data_avg_chunk_size" class="doc doc-headin
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.data_avg_chunk_size" class="headerlink" title="Permanent link">#</a></h4>
@@ -1829,8 +1829,8 @@ <h4 id="iscc_core.options.CoreOptions.instance_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.instance_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1859,8 +1859,8 @@ <h4 id="iscc_core.options.CoreOptions.mixed_bits" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.mixed_bits" class="headerlink" title="Permanent link">#</a></h4>
@@ -1889,8 +1889,8 @@ <h4 id="iscc_core.options.CoreOptions.io_read_size" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.io_read_size" class="headerlink" title="Permanent link">#</a></h4>
@@ -1919,8 +1919,8 @@ <h4 id="iscc_core.options.CoreOptions.cdc_gear" class="doc doc-heading">
 
   
   <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
       <small class="doc doc-label doc-label-instance-attribute"><code>instance-attribute</code></small>
+      <small class="doc doc-label doc-label-class-attribute"><code>class-attribute</code></small>
   </span>
 
 <a href="#iscc_core.options.CoreOptions.cdc_gear" class="headerlink" title="Permanent link">#</a></h4>
diff --git a/search/search_index.json b/search/search_index.json
index 5c2642e..682b25c 100755
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"ISCC - Codec &amp; Algorithms","text":"<p><code>iscc-core</code> is the reference implementation of the core algorithms of the ISCC (International Standard Content Code)</p>"},{"location":"#what-is-the-iscc","title":"What is the ISCC","text":"<p>The ISCC is a similarity preserving fingerprint and identifier for digital media assets.</p> <p>ISCCs are generated algorithmically from digital content, just like cryptographic hashes. However, instead of using a single cryptographic hash function to identify data only, the ISCC uses various algorithms to create a composite identifier that exhibits similarity-preserving properties (soft hash).</p> <p>The component-based structure of the ISCC identifies content at multiple levels of abstraction. Each component is self-describing, modular, and can be used separately or with others to aid in various content identification tasks. The algorithmic design supports content deduplication, database synchronization, indexing, integrity verification, timestamping, versioning, data provenance, similarity clustering, anomaly detection, usage tracking, allocation of royalties, fact-checking and general digital asset management use-cases.</p>"},{"location":"#what-is-iscc-core","title":"What is <code>iscc-core</code>","text":"<p><code>iscc-core</code> is a python based reference library of the core algorithms to create standard-compliant ISCC codes. It also a good reference for porting ISCC to other programming languages.</p> <p>Tip</p> <p>This is a low level reference implementation that does not inlcude features like mediatype detection, metadata extraction or file format specific content extraction. Please have a look at the iscc-sdk which adds those higher level features on top of the <code>iscc-core</code> library.</p>"},{"location":"#project-status","title":"Project Status","text":"<p>The ISCC is under development as ISO/CD 24138 - International Standard Content Code within ISO/TC 46/SC 9/WG 18.</p>"},{"location":"#iscc-architecture","title":"ISCC Architecture","text":""},{"location":"#iscc-maintypes","title":"ISCC MainTypes","text":"Idx Slug Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more components with common header"},{"location":"#installation","title":"Installation","text":"<p>Use the package manager pip to install <code>iscc-core</code>.</p> <pre><code>pip install iscc-core\n</code></pre>"},{"location":"#quick-start","title":"Quick Start","text":"<pre><code>import json\nimport iscc_core as ic\nmeta_code = ic.gen_meta_code(name=\"ISCC Test Document!\")\nprint(f\"Meta-Code:     {meta_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(meta_code['iscc'])}\\n\")\n# Extract text from file\nwith open(\"demo.txt\", \"rt\", encoding=\"utf-8\") as stream:\ntext = stream.read()\ntext_code = ic.gen_text_code_v0(text)\nprint(f\"Text-Code:     {text_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(text_code['iscc'])}\\n\")\n# Process raw bytes of textfile\nwith open(\"demo.txt\", \"rb\") as stream:\ndata_code = ic.gen_data_code(stream)\nprint(f\"Data-Code:     {data_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(data_code['iscc'])}\\n\")\nstream.seek(0)\ninstance_code = ic.gen_instance_code(stream)\nprint(f\"Instance-Code: {instance_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(instance_code['iscc'])}\\n\")\n# Combine ISCC-UNITs into ISCC-CODE\niscc_code = ic.gen_iscc_code(\n(meta_code[\"iscc\"], text_code[\"iscc\"], data_code[\"iscc\"], instance_code[\"iscc\"])\n)\n# Create convenience `Code` object from ISCC string\niscc_obj = ic.Code(iscc_code[\"iscc\"])\nprint(f\"ISCC-CODE:     {ic.iscc_normalize(iscc_obj.code)}\")\nprint(f\"Structure:     {iscc_obj.explain}\")\nprint(f\"Multiformat:   {iscc_obj.mf_base32}\\n\")\n# Compare with changed ISCC-CODE:\nnew_dc, new_ic = ic.Code.rnd(mt=ic.MT.DATA), ic.Code.rnd(mt=ic.MT.INSTANCE)\nnew_iscc = ic.gen_iscc_code((meta_code[\"iscc\"], text_code[\"iscc\"], new_dc.uri, new_ic.uri))\nprint(f\"Compare ISCC-CODES:\\n{iscc_obj.uri}\\n{new_iscc['iscc']}\")\nprint(json.dumps(ic.iscc_compare(iscc_obj.code, new_iscc[\"iscc\"]), indent=2))\n</code></pre> <p>The output of this example is as follows:</p> <pre><code>Meta-Code:     ISCC:AAAT4EBWK27737D2\nStructure:     META-NONE-V0-64-3e103656bffdfc7a\n\nText-Code:     ISCC:EAAQMBEYQF6457DP\nStructure:     CONTENT-TEXT-V0-64-060498817dcefc6f\n\nData-Code:     ISCC:GAA7UJMLDXHPPENG\nStructure:     DATA-NONE-V0-64-fa258b1dcef791a6\n\nInstance-Code: ISCC:IAA3Y7HR2FEZCU4N\nStructure:     INSTANCE-NONE-V0-64-bc7cf1d14991538d\n\nISCC-CODE:     ISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nStructure:     ISCC-TEXT-V0-MCDI-3e103656bffdfc7a060498817dcefc6ffa258b1dcef791a6bc7cf1d14991538d\nMultiformat:   bzqavabj6ca3fnp757r5ambeyqf6457dp7isywhoo66i2npd46hiutektru\n\nCompare ISCC-CODES:\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G7Y7HA2BMECKMVRBEQXR2BJOS6NA\n{\n  \"meta_dist\": 0,\n  \"content_dist\": 0,\n  \"data_dist\": 33,\n  \"instance_match\": false\n}\n</code></pre>"},{"location":"#documentation","title":"Documentation","text":"<p>Documentation is published athttps://core.iscc.codes</p>"},{"location":"#development","title":"Development","text":"<p>Requirements</p> <ul> <li>Python 3.7.2 or higher for code generation and static site building.</li> <li>Poetry for installation and dependency management.</li> </ul> <p>Development Setup</p> <pre><code>git clone https://github.com/iscc/iscc-core.git\ncd iscc-core\npoetry install\n</code></pre> <p>Development Tasks</p> <p>Tests, coverage, code formatting and other tasks can be run with the <code>poe</code> command:</p> <pre><code>poe\n\nPoe the Poet - A task runner that works well with poetry.\nversion 0.18.1\n\nResult: No task specified.\n\nUSAGE\n  poe [-h] [-v | -q] [--root PATH] [--ansi | --no-ansi] task [task arguments]\nGLOBAL OPTIONS\n  -h, --help     Show this help page and exit\n--version      Print the version and exit\n-v, --verbose  Increase command output (repeatable)\n-q, --quiet    Decrease command output (repeatable)\n-d, --dry-run  Print the task contents but don't actually run it\n  --root PATH    Specify where to find the pyproject.toml\n  --ansi         Force enable ANSI output\n  --no-ansi      Force disable ANSI output\nCONFIGURED TASKS\n  gentests       Generate conformance test data\n  format         Code style formating with black\n  docs           Copy README.md to /docs\n  format-md      Markdown formating with mdformat\n  lf             Convert line endings to lf\n  test           Run tests with coverage\n  sec            Security check with bandit\n  all\n</code></pre> <p>Use <code>poe all</code> to run all tasks before committing any changes.</p>"},{"location":"#maintainers","title":"Maintainers","text":"<p>@titusz</p>"},{"location":"#contributing","title":"Contributing","text":"<p>Pull requests are welcome. For significant changes, please open an issue first to discuss your plans. Please make sure to update tests as appropriate.</p> <p>You may also want join our developer chat on Telegram at https://t.me/iscc_dev.</p>"},{"location":"changelog/","title":"Changelog","text":""},{"location":"changelog/#105-2023-12-07","title":"[1.0.5] - 2023-12-07","text":"<ul> <li>Improved simhash performance</li> <li>Added native dct and wtahash support</li> <li>Added Python 3.12 support</li> <li>Updated and relax dependencies</li> </ul>"},{"location":"changelog/#104-2023-06-05","title":"[1.0.4] - 2023-06-05","text":"<ul> <li>Removed bases dependency</li> <li>Fixed mkdocstrings</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#103-2023-03-12","title":"[1.0.3] - 2023-03-12","text":"<ul> <li>Fix binary wheels</li> </ul>"},{"location":"changelog/#102-2023-03-12","title":"[1.0.2] - 2023-03-12","text":"<ul> <li>Publish binary wheels</li> </ul>"},{"location":"changelog/#101-2023-03-11","title":"[1.0.1] - 2023-03-11","text":"<ul> <li>Switch to standard bitarray module</li> <li>Switch tests to latest environments</li> <li>Add Python 3.11 support to TROVE classifiers</li> </ul>"},{"location":"changelog/#100-2023-01-24","title":"[1.0.0] - 2023-01-24","text":"<ul> <li>ISO/CD 24138 v1 Release</li> <li>Updateted dependencies</li> </ul>"},{"location":"changelog/#0214-2023-01-17","title":"[0.2.14] - 2023-01-17","text":"<ul> <li>Added ISO Reference documentation</li> <li>Removed non-standard conformance tests</li> <li>Fixed Meta-Code documentation</li> <li>Improved documentation CSS</li> </ul>"},{"location":"changelog/#0213-2023-01-16","title":"[0.2.13] - 2023-01-16","text":"<ul> <li>Added documentation for options</li> <li>Added python 3.11 support</li> <li>Added Markdown formating</li> <li>Added developmnet documentation</li> <li>Updated architecture figures</li> <li>Documentation cleanup</li> <li>Improved example code</li> <li>Optimized LF conversion</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0212-2022-11-24","title":"[0.2.12] - 2022-11-24","text":"<ul> <li>Fixed issue with data url compound media types</li> <li>Added ISCC version validation to <code>iscc_validate</code></li> <li>Added prefix check to <code>iscc_normalize</code></li> <li>Bundled fonts with documentation</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0211-2022-07-03","title":"[0.2.11] - 2022-07-03","text":"<ul> <li>Add support for gracefull build failures</li> </ul>"},{"location":"changelog/#0210-2022-07-03","title":"[0.2.10] - 2022-07-03","text":"<ul> <li>Fix pip instalation is missing setuptools</li> <li>Update mkdocs</li> </ul>"},{"location":"changelog/#029-2022-07-03","title":"[0.2.9] - 2022-07-03","text":"<ul> <li>Added iscc_compare function</li> <li>Optimized soft_hash_audio performance</li> <li>Removed Cython from build requirements</li> <li>Fixed api listing tool</li> <li>Updated codec architecture figure</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#028-2022-04-21","title":"[0.2.8] - 2022-04-21","text":"<ul> <li>Fixed bug with subtype for semantic code</li> <li>Changed URI representation to upper case</li> <li>Changed to disallow ISCC-ID creation from ISCC-IDs</li> <li>Added line conversion tool</li> <li>Removed source wheel distribution</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#027-2022-04-16","title":"[0.2.7] - 2022-04-16","text":"<ul> <li>Fixed bug in iscc_id_incr_v0</li> <li>Added support to accept ISCC-ID URI as input for iscc_id_incr_v0</li> <li>Added guard against custom subtype in random ISCC-CODE generation.</li> </ul>"},{"location":"changelog/#026-2022-04-13","title":"[0.2.6] - 2022-04-13","text":"<ul> <li>Added <code>KY</code> and <code>MM</code> to valid prefixes</li> <li>Added support to check for compiled extension modules</li> <li>Added universal wheel distribution</li> </ul>"},{"location":"changelog/#025-2022-04-10","title":"[0.2.5] - 2022-04-10","text":"<ul> <li>Fixed missing <code>jcs</code> dependency</li> <li>Added SubType <code>NONE</code> to MT.ISCC to distinquish from SUM</li> <li>Added support for deterministic generation of random ISCC-CODEs</li> <li>Added support for custom bit-sizes for random ISCC-CODEs</li> <li>Moved changelog into separate file</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#024-2022-03-19","title":"[0.2.4] - 2022-03-19","text":"<ul> <li>Updated dependencies</li> <li>Added Flake.from_int and Flake.from_string</li> <li>Made Flake comparable and hashable</li> <li>Use standard hex encoded multihash for datahash and metahash</li> </ul>"},{"location":"changelog/#023-2022-03-06","title":"[0.2.3] - 2022-03-06","text":"<ul> <li>Update to iscc-schema 0.3.3</li> <li>Change image normalization instructions</li> <li>Fix issue with exporting cdc cython only functions</li> </ul>"},{"location":"changelog/#021-2022-03-03","title":"[0.2.1] - 2022-03-03","text":"<ul> <li>Cleanup and update dependencies</li> <li>Fix bitarray api change</li> <li>Fix developer commands</li> </ul>"},{"location":"changelog/#020-2022-02-24","title":"[0.2.0] - 2022-02-24","text":"<ul> <li>Complete API refactoring</li> <li>Use Data-URL as input for Meta-Code</li> <li>Use wallet address for ISCC-ID creation</li> <li>Added new Flake-Code (distributed time/random ID)</li> <li>Replaced assertions with exeptions</li> <li>Use secure random functions</li> <li>Retired Python 3.6 support (EOL)</li> <li>Return simple <code>dict</code> objects from generator functions</li> <li>Added ISCC string validation</li> <li>Added multiple helper functions</li> </ul>"},{"location":"changelog/#019-2021-12-17","title":"[0.1.9] - 2021-12-17","text":"<ul> <li>Added warning on non-standard options</li> <li>Added multiformats support</li> <li>Added uri representation</li> <li>Removed redundant cdc_avg_chunk_size option</li> <li>Updated codec format documentation</li> </ul>"},{"location":"changelog/#018-2021-12-12","title":"[0.1.8] - 2021-12-12","text":"<ul> <li>Added conformance tests for all top level functions</li> <li>Added conformance tests to source dir</li> <li>Added conformance module with <code>selftest</code> function</li> <li>Changed gen_image_code to accept normalized pixels instead of stream</li> <li>Changed opts to core_opts</li> <li>Removed image pre-processing and Pillow dependency</li> <li>Fixed readability of conformance tests</li> <li>Fixed soft_hash_video_v0 to accept non-tuple sequences</li> <li>Updated example code</li> </ul>"},{"location":"changelog/#017-2021-12-09","title":"[0.1.7] - 2021-12-09","text":"<ul> <li>Add dotenv for enviroment based configuration</li> <li>Cleanup package toplevel imports</li> <li>Return schema objects for iscc_code and iscc_id</li> <li>Exclude unset and none values from result dicts</li> <li>Add support for multiple code combinations for ISCC-CODE</li> <li>Add support for ISCC-ID based on singular Instance-Code</li> <li>Add initial conformance test system</li> </ul>"},{"location":"changelog/#016-2021-11-29","title":"[0.1.6] - 2021-11-29","text":"<ul> <li>Show counter for ISCC-ID in Code.explain</li> </ul>"},{"location":"changelog/#015-2021-11-28","title":"[0.1.5] - 2021-11-28","text":"<ul> <li>Fix documentation</li> <li>Change metahash creation logic</li> <li>Refactor models</li> <li>Add Content-Code-Mixed</li> <li>Add ISCC-ID</li> <li>Refactor <code>compose</code> to <code>gen_iscc_code</code></li> <li>Refactor <code>models</code> to <code>schema</code></li> </ul>"},{"location":"changelog/#014-2021-11-17","title":"[0.1.4] - 2021-11-17","text":"<ul> <li>Simplified options</li> <li>Optimize video WTA-hash for use with 64-bit granular features</li> </ul>"},{"location":"changelog/#013-2021-11-15","title":"[0.1.3] - 2021-11-15","text":"<ul> <li>Try to compile Cython/C accelerator modules when installing via pip</li> <li>Simplify soft_hash api return values</li> <li>Add .code() method to InstanceHasher, DataHasher</li> <li>Remove granular fingerprint calculation</li> <li>Add more top-level imports</li> </ul>"},{"location":"changelog/#012-2021-11-14","title":"[0.1.2] - 2021-11-14","text":"<ul> <li>Export more functions to toplevel</li> <li>Return schema driven objects from ISCC code generators.</li> </ul>"},{"location":"changelog/#011-2021-11-14","title":"[0.1.1] - 2021-11-14","text":"<ul> <li>Fix packaging problems</li> </ul>"},{"location":"changelog/#010-2021-11-13","title":"[0.1.0] - 2021-11-13","text":"<ul> <li>Initial release</li> </ul>"},{"location":"conformance/","title":"ISCC - Conformance Testing","text":"<p>An application that claims ISCC conformance MUST pass all core functions from the ISCC conformance test suite. The test suite is available as JSON data on Github. Test data is structured as follows:</p> <pre><code>{\n\"&lt;function_name&gt;\": {\n\"&lt;test_name&gt;\": {\n\"inputs\": [\"&lt;value1&gt;\", \"&lt;value2&gt;\"],\n\"outputs\": [\"value1&gt;\", \"&lt;value2&gt;\"]\n}\n}\n}\n</code></pre> <p>Inputs that are expected to be <code>raw bytes or byte-streams</code> are embedded as HEX encoded strings in JSON and prefixed with <code>stream:</code> or <code>bytes</code> to support automated decoding during implementation testing.</p> <p>Example</p> <p>Byte-stream outputs in JSON test data: <pre><code>\"gen_data_code_v0\": {\n\"test_0000_two_bytes_64\": {\n\"inputs\": [\n\"stream:ff00\",\n64\n],\n\"outputs\": {\n\"iscc\": \"GAAXL2XYM5BQIAZ3\"\n}\n},\n...\n</code></pre></p>"},{"location":"conformance/#iscc_core.conformance.conformance_testdata","title":"<code>conformance_testdata()</code>","text":"<p>Yield tuples of test data.</p> <p>Returns:</p> Type Description <code>Generator[Tuple[str, Callable, List[Any], List[Any]]]</code> <p>Tuple with testdata (test_name, func_obj, inputs, outputs)</p>"},{"location":"conformance/#iscc_core.conformance.conformance_selftest","title":"<code>conformance_selftest()</code>","text":"<p>Run conformance tests.</p> <p>Returns:</p> Type Description <code>bool</code> <p>whether all tests passed</p>"},{"location":"constants/","title":"ISCC - Types and Constants","text":""},{"location":"constants/#iscc_core.constants.MT","title":"<code>MT</code>","text":""},{"location":"constants/#iscc_core.constants.MT--mt-maintypes","title":"MT - MainTypes","text":"Uint Symbol Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more ISCC-UNITs with common header"},{"location":"constants/#iscc_core.constants.ST","title":"<code>ST</code>","text":""},{"location":"constants/#iscc_core.constants.ST--st-subtypes","title":"ST - SubTypes","text":"Uint Symbol Bits Purpose 0 NONE 0000 For MainTypes that do not specify SubTypes"},{"location":"constants/#iscc_core.constants.ST_CC","title":"<code>ST_CC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_CC--st_cc","title":"ST_CC","text":"<p>SubTypes for <code>MT.CONTENT</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Match on syntactic text similarity 1 IMAGE 0001 Match on perceptual image similarity 2 AUDIO 0010 Match on audio chroma similarity 3 VIDEO 0011 Match on perceptual similarity 4 MIXED 0100 Match on similarity of content codes"},{"location":"constants/#iscc_core.constants.ST_ISCC","title":"<code>ST_ISCC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_ISCC--st_iscc","title":"ST_ISCC","text":"<p>SubTypes for <code>MT.ISCC</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Composite ISCC inlcuding Text-Code 1 IMAGE 0001 Composite ISCC inlcuding Image-Code 2 AUDIO 0010 Composite ISCC inlcuding Audio-Code 3 VIDEO 0011 Composite ISCC inlcuding Video-Code 4 MIXED 0100 Composite ISCC inlcuding Mixed-Code 5 SUM 0101 Composite ISCC inlcuding only Data- and Instance-Code 6 NONE 0110 Composite ISCC including Meta, Data and Instance-Code"},{"location":"constants/#iscc_core.constants.VS","title":"<code>VS</code>","text":""},{"location":"constants/#iscc_core.constants.VS--vs-version","title":"VS - Version","text":"<p>Code Version</p> Uint Symbol Bits Purpose 0 V0 0000 Initial Version of Code without breaking changes"},{"location":"constants/#iscc_core.constants.LN","title":"<code>LN</code>","text":""},{"location":"constants/#iscc_core.constants.LN--ln-length","title":"LN - Length","text":"<p>Valid lengths for hash-digests.</p> <ul> <li>L32 = 32</li> <li>L64 = 64</li> <li>L72 = 72</li> <li>L80 = 80</li> <li>L96 = 96</li> <li>L128 = 128</li> <li>L160 = 160</li> <li>L192 = 192</li> <li>L224 = 224</li> <li>L256 = 256</li> </ul>"},{"location":"constants/#iscc_core.constants.MULTIBASE","title":"<code>MULTIBASE</code>","text":"<p>Supported Multibase encodings.</p> <ul> <li>base16 -&gt; f</li> <li>base32 -&gt; b</li> <li>base32hex -&gt; v</li> <li>base58btc -&gt; z</li> <li>base64url -&gt; u</li> </ul>"},{"location":"iscc_code/","title":"ISCC-CODE","text":"<p>A multi-component identifier for digital media assets.</p> <p>An ISCC-CODE can be generated from the concatenation of the digests of the following five ISCC-UNITs together with a single common header:</p> <ul> <li>Meta-Code - Encodes metadata similarity</li> <li>Semantic-Code - Encodes semantic content similarity (to be developed)</li> <li>Content-Code - Encodes syntactic/perceptual similarity</li> <li>Data-Code - Encodes raw bitstream similarity</li> <li>Instance-Code - Data checksum</li> </ul> <p>The following sequences of ISCC-UNITs are possible:</p> <ul> <li>Data, Instance</li> <li>Content, Data, Instance</li> <li>Semantic, Data, Instance</li> <li>Content, Semantic, Data, Instance</li> <li>Meta, Data, Instance</li> <li>Meta, Content, Data, Instance</li> <li>Meta, Semantic, Data, Instance</li> <li>Meta, Semantic, Content, Data, Instance</li> </ul>"},{"location":"iscc_code/#iscc_core.iscc_code.gen_iscc_code_v0","title":"<code>gen_iscc_code_v0(codes)</code>","text":"<p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iscc_id/","title":"ISCC-ID","text":"<p>A decentralized, owned, and short identifier for digital assets.</p> <p>The ISCC-ID is generated from a similarity-hash of the units of an ISCC-CODE together with a blockchain wallet address. Its SubType designates the blockchain from which the ISCC-ID was minted. The similarity-hash is always at least 64-bits and optionally suffixed with a <code>uvarint</code> endcoded <code>uniqueness counter</code>. The <code>uniqueness counter</code> is added and incremented only if the mint colides with a pre-existing ISCC-ID minted from the same blockchain from a different ISCC-CODE or from an identical ISCC-CODE registered by a different signatory.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id","title":"<code>gen_iscc_id(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate  ISCC-ID from ISCC-CODE with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id_v0","title":"<code>gen_iscc_id_v0(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate an ISCC-ID from an ISCC-CODE with uniqueness counter 'uc' with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.soft_hash_iscc_id_v0","title":"<code>soft_hash_iscc_id_v0(iscc_code, wallet, uc = 0)</code>","text":"<p>Calculate ISCC-ID hash digest from ISCC-CODE with algorithm v0.</p> <p>Accepts an ISCC-CODE or any sequence of ISCC-UNITs.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>ISCC-CODE</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter for ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Digest for ISCC-ID without header but including uniqueness counter.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr","title":"<code>iscc_id_incr(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr_v0","title":"<code>iscc_id_incr_v0(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one (without \"ISCC:\" prefix).</p>"},{"location":"iscc_id/#iscc_core.iscc_id.alg_simhash_from_iscc_id","title":"<code>alg_simhash_from_iscc_id(iscc_id, wallet)</code>","text":"<p>Extract similarity preserving hex-encoded hash digest from ISCC-ID</p> <p>We need to un-xor the ISCC-ID hash digest with the wallet address hash to obtain the similarity preserving bytestring.</p>"},{"location":"iso-reference/","title":"ISCC - ISO Reference","text":"<p>The following functions are the reference implementations of ISO 24138:</p>"},{"location":"iso-reference/#iso-24138-51-meta-code","title":"ISO 24138 / 5.1 Meta-Code","text":"<code>gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits)</code> # <p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"iso-reference/#iso-24138-53-text-code","title":"ISO 24138 / 5.3 Text-Code","text":"<code>gen_text_code_v0(text, bits=ic.core_opts.text_bits)</code> # <p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"iso-reference/#iso-24138-54-image-code","title":"ISO 24138 / 5.4 Image-Code","text":"<code>gen_image_code_v0(pixels, bits=ic.core_opts.image_bits)</code> # <p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p> Source code in <code>iscc_core\\code_content_image.py</code> <pre><code>def gen_image_code_v0(pixels, bits=ic.core_opts.image_bits):\n# type: (Sequence[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Image with algorithm v0.\n    :param Sequence[int] pixels: Normalized image pixels (32x32 flattened gray values)\n    :param int bits: Bit-length of ISCC Content-Code Image (default 64).\n    :return: ISCC object with Content-Code Image.\n    :rtype: ISCC\n    \"\"\"\ndigest = soft_hash_image_v0(pixels, bits=bits)\nimage_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.IMAGE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + image_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-55-audio-code","title":"ISO 24138 / 5.5 Audio-Code","text":"<code>gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits)</code> # <p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p> Source code in <code>iscc_core\\code_content_audio.py</code> <pre><code>def gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits):\n# type: (Iterable[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Audio with algorithm v0.\n    :param Iterable[int] cv: Chromaprint vector\n    :param int bits: Bit-length resulting Content-Code Audio (multiple of 64)\n    :return: ISCC object with Content-Code Audio\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_audio_v0(cv, bits=bits)\naudio_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.AUDIO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + audio_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-56-video-code","title":"ISO 24138 / 5.6 Video-Code","text":"<code>gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits)</code> # <p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p> Source code in <code>iscc_core\\code_content_video.py</code> <pre><code>def gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits):\n# type: (Sequence[ic.FrameSig], int) -&gt; dict\n\"\"\"\n    Create an ISCC Video-Code with algorithm v0.\n    :param ic.FrameSig frame_sigs: Sequence of MP7 frame signatures\n    :param int bits: Bit-length resulting Video-Code (multiple of 64)\n    :return: ISCC object with Video-Code\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_video_v0(frame_sigs, bits=bits)\nvideo_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.VIDEO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + video_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-57-mixed-code","title":"ISO 24138 / 5.7 Mixed-Code","text":"<code>gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits)</code> # <p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p> Source code in <code>iscc_core\\code_content_mixed.py</code> <pre><code>def gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits):\n# type: (Sequence[str], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code-Mixed with algorithm v0.\n    If the provided codes are of mixed length they are stripped to `bits` length for\n    calculation.\n    :param Iterable[str] codes: a list of Content-Codes.\n    :param int bits: Target bit-length of generated Content-Code-Mixed.\n    :return: ISCC object with Content-Code Mixed.\n    :rtype: dict\n    \"\"\"\ndigests = [ic.decode_base32(ic.iscc_clean(code)) for code in codes]\ndigest = soft_hash_codes_v0(digests, bits=bits)\nmixed_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.MIXED,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + mixed_code\nreturn dict(iscc=iscc, parts=list(codes))\n</code></pre>"},{"location":"iso-reference/#iso-24138-58-data-code","title":"ISO 24138 / 5.8 Data-Code","text":"<code>gen_data_code_v0(stream, bits=ic.core_opts.data_bits)</code> # <p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p> Source code in <code>iscc_core\\code_data.py</code> <pre><code>def gen_data_code_v0(stream, bits=ic.core_opts.data_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Data-Code with algorithm v0.\n    :param Stream stream: Input data stream.\n    :param int bits: Bit-length of ISCC Data-Code (default 64).\n    :return: ISCC object with Data-Code\n    :rtype: dict\n    \"\"\"\nhasher = DataHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ndata_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + data_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-59-instance-code","title":"ISO 24138 / 5.9 Instance-Code","text":"<code>gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits)</code> # <p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p> Source code in <code>iscc_core\\code_instance.py</code> <pre><code>def gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Instance-Code with algorithm v0.\n    :param Stream stream: Binary data stream for Instance-Code generation\n    :param int bits: Bit-length of resulting Instance-Code (multiple of 64)\n    :return: ISCC object with Instance-Code and properties: datahash, filesize\n    :rtype: dict\n    \"\"\"\nhasher = InstanceHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ninstance_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + instance_code\ninstance_code_obj = dict(\niscc=iscc,\ndatahash=hasher.multihash(),\nfilesize=hasher.filesize,\n)\nreturn instance_code_obj\n</code></pre>"},{"location":"iso-reference/#iso-24138-60-iscc-code","title":"ISO 24138 / 6.0 ISCC-CODE","text":"<code>gen_iscc_code_v0(codes)</code> # <p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"algorithms/cdc/","title":"ISCC - Content Defined Chunking","text":"<p>Compatible with fastcdc</p>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_chunks","title":"<code>alg_cdc_chunks(data, utf32, avg_chunk_size = ic.core_opts.data_avg_chunk_size)</code>","text":"<p>A generator that yields data-dependent chunks for <code>data</code>.</p> <p>Usage Example:</p> <pre><code>for chunk in cdc_data_chunks(data):\nhash(chunk)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>Raw data for variable sized chunking.</p> required <code>utf32</code> <code>bool</code> <p>If true assume we are chunking text that is utf32 encoded.</p> required <code>avg_chunk_size</code> <code>int</code> <p>Target chunk size in number of bytes.</p> <code>ic.core_opts.data_avg_chunk_size</code> <p>Returns:</p> Type Description <code>Generator[bytes]</code> <p>A generator that yields data chunks of variable sizes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_chunks(data, utf32, avg_chunk_size=ic.core_opts.data_avg_chunk_size):\n# type: (Data, bool, int) -&gt; Generator[bytes, None, None]\n\"\"\"\n    A generator that yields data-dependent chunks for `data`.\n    Usage Example:\n    ```python\n    for chunk in cdc_data_chunks(data):\n        hash(chunk)\n    ```\n    :param bytes data: Raw data for variable sized chunking.\n    :param bool utf32: If true assume we are chunking text that is utf32 encoded.\n    :param int avg_chunk_size: Target chunk size in number of bytes.\n    :return: A generator that yields data chunks of variable sizes.\n    :rtype: Generator[bytes]\n    \"\"\"\nstream = io.BytesIO(data)\nbuffer = stream.read(ic.core_opts.io_read_size)\nif not buffer:\nyield b\"\"\nmi, ma, cs, mask_s, mask_l = alg_cdc_params(avg_chunk_size)\nbuffer = memoryview(buffer)\nwhile buffer:\nif len(buffer) &lt;= ma:\nbuffer = memoryview(bytes(buffer) + stream.read(ic.core_opts.io_read_size))\ncut_point = alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)\n# Make sure cut points are at 4-byte aligned for utf32 encoded text\nif utf32:\ncut_point -= cut_point % 4\nyield bytes(buffer[:cut_point])\nbuffer = buffer[cut_point:]\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_offset","title":"<code>alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)</code>","text":"<p>Find breakpoint offset for a given buffer.</p> <p>Parameters:</p> Name Type Description Default <code>buffer</code> <code>Data</code> <p>The data to be chunked.</p> required <code>mi</code> <code>int</code> <p>Minimum chunk size.</p> required <code>ma</code> <code>int</code> <p>Maximung chunk size.</p> required <code>cs</code> <code>int</code> <p>Center size.</p> required <code>mask_s</code> <code>int</code> <p>Small mask.</p> required <code>mask_l</code> <code>int</code> <p>Large mask.</p> required <p>Returns:</p> Type Description <code>int</code> <p>Offset of dynamic cutpoint in number of bytes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l):\n# type: (ic.Data, int, int, int, int, int) -&gt; int\n\"\"\"\n    Find breakpoint offset for a given buffer.\n    :param Data buffer: The data to be chunked.\n    :param int mi: Minimum chunk size.\n    :param int ma: Maximung chunk size.\n    :param int cs: Center size.\n    :param int mask_s: Small mask.\n    :param int mask_l: Large mask.\n    :return: Offset of dynamic cutpoint in number of bytes.\n    :rtype: int\n    \"\"\"\npattern = 0\ni = mi\nsize = len(buffer)\nbarrier = min(cs, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_s:\nreturn i + 1\ni += 1\nbarrier = min(ma, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_l:\nreturn i + 1\ni += 1\nreturn i\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_params","title":"<code>alg_cdc_params(avg_size: int) -&gt; tuple</code>","text":"<p>Calculate CDC parameters</p> <p>Parameters:</p> Name Type Description Default <code>avg_size</code> <code>int</code> <p>Target average size of chunks in number of bytes.</p> required <p>Returns:</p> Type Description <code>tuple</code> <p>Tuple of (min_size, max_size, center_size, mask_s, mask_l).</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_params(avg_size: int) -&gt; tuple:\n\"\"\"\n    Calculate CDC parameters\n    :param int avg_size: Target average size of chunks in number of bytes.\n    :returns: Tuple of (min_size, max_size, center_size, mask_s, mask_l).\n    \"\"\"\nceil_div = lambda x, y: (x + y - 1) // y\nmask = lambda b: 2**b - 1\nmin_size = avg_size // 4\nmax_size = avg_size * 8\noffset = min_size + ceil_div(min_size, 2)\ncenter_size = avg_size - offset\nbits = round(log2(avg_size))\nmask_s = mask(bits + 1)\nmask_l = mask(bits - 1)\nreturn min_size, max_size, center_size, mask_s, mask_l\n</code></pre>"},{"location":"algorithms/dct/","title":"ISCC - Discrete Cosine Transform","text":""},{"location":"algorithms/dct/#iscc_core.dct.alg_dct","title":"<code>alg_dct(v)</code>","text":"<p>Discrete cosine transform.</p> <p>See: nayuki.io.</p> <p>Parameters:</p> Name Type Description Default <code>v</code> <code>Sequence[float]</code> <p>Input vector for DCT calculation.</p> required <p>Returns:</p> Type Description <code>List</code> <p>DCT Transformed vector.</p> Source code in <code>iscc_core\\dct.py</code> <pre><code>def alg_dct(v):\n# type: (Sequence[float]) -&gt; List\n\"\"\"\n    Discrete cosine transform.\n    See: [nayuki.io](https://www.nayuki.io/page/fast-discrete-cosine-transform-algorithms).\n    :param Sequence[float] v: Input vector for DCT calculation.\n    :return: DCT Transformed vector.\n    :rtype: List\n    \"\"\"\nn = len(v)\nif n == 1:\nreturn list(v)\nelif n == 0 or n % 2 != 0:\nraise ValueError()\nelse:\nhalf = n // 2\nalpha = [(v[i] + v[-(i + 1)]) for i in range(half)]\nbeta = [\n(v[i] - v[-(i + 1)]) / (math.cos((i + 0.5) * math.pi / n) * 2.0) for i in range(half)\n]\nalpha = alg_dct(alpha)\nbeta = alg_dct(beta)\nresult = []\nfor i in range(half - 1):\nresult.append(alpha[i])\nresult.append(beta[i] + beta[i + 1])\nresult.append(alpha[-1])\nresult.append(beta[-1])\nreturn result\n</code></pre>"},{"location":"algorithms/minhash/","title":"ISCC - Minhash","text":""},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash","title":"<code>alg_minhash(features)</code>","text":"<p>Calculate a 64 dimensional minhash integer vector.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>List[int]</code> <p>Minhash vector</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash(features):\n# type: (List[int]) -&gt; List[int]\n\"\"\"\n    Calculate a 64 dimensional minhash integer vector.\n    :param List[int] features: List of integer features\n    :return: Minhash vector\n    :rtype: List[int]\n    \"\"\"\nreturn [\nmin([(((a * f + b) &amp; MAXI64) % MPRIME) &amp; MAXH for f in features]) for a, b in zip(MPA, MPB)\n]\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_64","title":"<code>alg_minhash_64(features)</code>","text":"<p>Create 64-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>64-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_64(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 64-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 64-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 1)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_256","title":"<code>alg_minhash_256(features)</code>","text":"<p>Create 256-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_256(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 256-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 4)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_compress","title":"<code>alg_minhash_compress(mhash, lsb = 4)</code>","text":"<p>Compress minhash vector to byte hash-digest.</p> <p>Concatenates <code>lsb</code> number of least-significant bits from each integer in <code>mhash</code>. For example an <code>mhash</code> with 64 integers and <code>lsb=4</code> will produce a 256-bit summary of the minhash vector.</p> <p>Parameters:</p> Name Type Description Default <code>mhash</code> <code>List[int]</code> <p>List of minhash integer features</p> required <code>lsb</code> <code>int</code> <p>Number of the least significant bits to retain</p> <code>4</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_compress(mhash, lsb=4):\n# type: (List[int], int) -&gt; bytes\n\"\"\"\n    Compress minhash vector to byte hash-digest.\n    Concatenates `lsb` number of least-significant bits from each integer in `mhash`.\n    For example an `mhash` with 64 integers and `lsb=4` will produce a 256-bit summary\n    of the minhash vector.\n    :param List[int] mhash: List of minhash integer features\n    :param int lsb: Number of the least significant bits to retain\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nbits: str = \"\"\nfor bitpos in range(lsb):\nfor h in mhash:\nbits += str(h &gt;&gt; bitpos &amp; 1)\nreturn int(bits, 2).to_bytes((len(bits) + 7) // 8, \"big\")\n</code></pre>"},{"location":"algorithms/simhash/","title":"ISCC - Simhash","text":""},{"location":"algorithms/simhash/#iscc_core.simhash.alg_simhash","title":"<code>alg_simhash(hash_digests)</code>","text":"<p>Creates a similarity preserving hash from a sequence of equal sized hash digests.</p> <p>Parameters:</p> Name Type Description Default <code>hash_digests</code> <code>list</code> <p>A sequence of equaly sized byte-hashes.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity byte-hash</p> Source code in <code>iscc_core\\simhash.py</code> <pre><code>def alg_simhash(hash_digests):\n# type: (list[bytes]) -&gt; bytes\n\"\"\"\n    Creates a similarity preserving hash from a sequence of equal sized hash digests.\n    :param list hash_digests: A sequence of equaly sized byte-hashes.\n    :returns: Similarity byte-hash\n    :rtype: bytes\n    \"\"\"\nn_bytes = len(hash_digests[0])\nn_bits = n_bytes * 8\nvector = [0] * n_bits\nfor digest in hash_digests:\nh = bitarray()\nh.frombytes(digest)\nfor i in range(n_bits):\nvector[i] += h[i]\nminfeatures = len(hash_digests) / 2\nshash = 0\nfor i in range(n_bits):\nif vector[i] &gt;= minfeatures:\nshash |= 1 &lt;&lt; (n_bits - 1 - i)\nreturn shash.to_bytes(n_bytes, \"big\")\n</code></pre>"},{"location":"algorithms/wtahash/","title":"ISCC - Winner Takes All Hash","text":""},{"location":"algorithms/wtahash/#iscc_core.wtahash.alg_wtahash","title":"<code>alg_wtahash(vec: Sequence[float], bits: Sequence[float]) -&gt; bytes</code>","text":"<p>Calculate WTA Hash for vector with 380 values (MP7 frame signature).</p> Source code in <code>iscc_core\\wtahash.py</code> <pre><code>def alg_wtahash(vec: Sequence[float], bits) -&gt; bytes:\n\"\"\"Calculate WTA Hash for vector with 380 values (MP7 frame signature).\"\"\"\nh = []\nfor perm in WTA_VIDEO_ID_PERMUTATIONS:\nv = vec[perm[0]], vec[perm[1]]\nh.append(v.index(max(v)))\nif len(h) == bits:\nbreak\nh = bitarray(h).tobytes()\nreturn h\n</code></pre>"},{"location":"codec/","title":"ISCC - Codec","text":"<p>This module implements encoding, decoding and transcoding functions of ISCC</p>"},{"location":"codec/#codec-overview","title":"Codec Overview","text":""},{"location":"codec/#codec-functions","title":"Codec Functions","text":""},{"location":"codec/#iscc_core.codec.encode_component","title":"<code>encode_component(mtype, stype, version, bit_length, digest)</code>","text":"<p>Encode an ISCC unit inlcuding header and body with standard base32 encoding.</p> <p>Note</p> <p>The <code>length</code> value must be the length in number of bits for the component. If <code>digest</code> has more bits than specified by <code>length</code> it wil be truncated.</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>Maintype of unit (0-6)</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit depending on MainType (0-5)</p> required <code>version</code> <code>Version</code> <p>Version of unit algorithm (0).</p> required <code>bit_length</code> <code>length</code> <p>Length of unit, in number of bits (multiple of 32)</p> required <code>digest</code> <code>bytes</code> <p>The hash digest of the unit.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32 encoded ISCC-UNIT.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_component(mtype, stype, version, bit_length, digest):\n# type: (MainType, SubType, Version, Length, bytes) -&gt; str\n\"\"\"\n    Encode an ISCC unit inlcuding header and body with standard base32 encoding.\n    !!! note\n        The `length` value must be the **length in number of bits** for the component.\n        If `digest` has more bits than specified by `length` it wil be truncated.\n    :param MainType mtype: Maintype of unit (0-6)\n    :param SubType stype: SubType of unit depending on MainType (0-5)\n    :param Version version: Version of unit algorithm (0).\n    :param length bit_length: Length of unit, in number of bits (multiple of 32)\n    :param bytes digest: The hash digest of the unit.\n    :return: Base32 encoded ISCC-UNIT.\n    :rtype: str\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.ID, MT.FLAKE):\nencoded_length = encode_length(mtype, bit_length)\nelif mtype == MT.ISCC:\nraise ValueError(f\"{mtype} is not a unit\")\nelse:\nraise ValueError(f\"Illegal MainType {mtype}\")\nnbytes = bit_length // 8\nheader = encode_header(mtype, stype, version, encoded_length)\nbody = digest[:nbytes]\ncomponent_code = encode_base32(header + body)\nreturn component_code\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_header","title":"<code>encode_header(mtype, stype, version = 0, length = 1)</code>","text":"<p>Encodes header values with nibble-sized (4-bit) variable-length encoding. The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles is uneven it is padded with 4-bit <code>0000</code> at the end.</p> <p>Warning</p> <p>The length value must be encoded beforhand because its semantics depend on the MainType (see <code>encode_length</code> function).</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>MainType of unit.</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit.</p> required <code>version</code> <code>Version</code> <p>Version of component algorithm.</p> <code>0</code> <code>length</code> <code>Length</code> <p>length value of unit (1 means 64-bits for standard units)</p> <code>1</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Varnibble stream encoded ISCC header as bytes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_header(mtype, stype, version=0, length=1):\n# type: (MainType, SubType, Version, Length) -&gt; bytes\n\"\"\"\n    Encodes header values with nibble-sized (4-bit) variable-length encoding.\n    The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles\n    is uneven it is padded with 4-bit `0000` at the end.\n    !!! warning\n        The length value must be encoded beforhand because its semantics depend on\n        the MainType (see `encode_length` function).\n    :param MainType mtype: MainType of unit.\n    :param SubType stype: SubType of unit.\n    :param Version version: Version of component algorithm.\n    :param Length length: length value of unit (1 means 64-bits for standard units)\n    :return: Varnibble stream encoded ISCC header as bytes.\n    :rtype: bytes\n    \"\"\"\n# TODO verify that all header params and there combination is valid\nheader = bitarray()\nfor n in (mtype, stype, version, length):\nheader += encode_varnibble(n)\n# Append zero-padding if required (right side, least significant bits).\nheader.fill()\nreturn header.tobytes()\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_header","title":"<code>decode_header(data)</code>","text":"<p>Decodes varnibble encoded header and returns it together with <code>tail data</code>.</p> <p>Tail data is included to enable decoding of sequential ISCCs. The returned tail data must be truncated to decode_length(r[0], r[3]) bits to recover the actual hash-bytes.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>ISCC bytes</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>(MainType, SubType, Version, length, TailData)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_header(data):\n# type: (bytes) -&gt; IsccTuple\n\"\"\"\n    Decodes varnibble encoded header and returns it together with `tail data`.\n    Tail data is included to enable decoding of sequential ISCCs. The returned tail\n    data must be truncated to decode_length(r[0], r[3]) bits to recover the actual\n    hash-bytes.\n    :param bytes data: ISCC bytes\n    :return: (MainType, SubType, Version, length, TailData)\n    :rtype: IsccTuple\n    \"\"\"\nresult = []\nba = bitarray()\nba.frombytes(data)\ndata = ba\nfor _ in range(4):\nvalue, data = decode_varnibble(data)\nresult.append(value)\n# Strip 4-bit padding if required\nif len(data) % 8 and data[:4] == bitarray(\"0000\"):\ndata = data[4:]\nresult.append(data.tobytes())\nreturn tuple(result)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_varnibble","title":"<code>encode_varnibble(n)</code>","text":"<p>Writes integer to variable length sequence of 4-bit chunks.</p> <p>Variable-length encoding scheme:</p> prefix bits nibbles data bits unsigned range 0 1 3 0 - 7 10 2 6 8 - 71 110 3 9 72 - 583 1110 4 12 584 - 4679 <p>Parameters:</p> Name Type Description Default <code>n</code> <code>int</code> <p>Positive integer to be encoded as varnibble (0-4679)</p> required <p>Returns:</p> Type Description <code>bitarray</code> <p>Varnibble encoded integera</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_varnibble(n):\n# type: (int) -&gt; bitarray\n\"\"\"\n    Writes integer to variable length sequence of 4-bit chunks.\n    Variable-length encoding scheme:\n    ------------------------------------------------------\n    | prefix bits | nibbles | data bits | unsigned range |\n    | ----------- | ------- | --------- | -------------- |\n    | 0           | 1       | 3         | 0 - 7          |\n    | 10          | 2       | 6         | 8 - 71         |\n    | 110         | 3       | 9         | 72 - 583       |\n    | 1110        | 4       | 12        | 584 - 4679     |\n    :param int n: Positive integer to be encoded as varnibble (0-4679)\n    :return: Varnibble encoded integera\n    :rtype: bitarray\n    \"\"\"\nif 0 &lt;= n &lt; 8:\nreturn int2ba(n, length=4)\nelif 8 &lt;= n &lt; 72:\nreturn bitarray(\"10\") + int2ba(n - 8, length=6)\nelif 72 &lt;= n &lt; 584:\nreturn bitarray(\"110\") + int2ba(n - 72, length=9)\nelif 584 &lt;= n &lt; 4680:\nreturn bitarray(\"1110\") + int2ba(n - 584, length=12)\nelse:\nraise ValueError(\"Value must be between 0 and 4679\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_varnibble","title":"<code>decode_varnibble(b)</code>","text":"<p>Reads first varnibble, returns its integer value and remaining bits.</p> <p>Parameters:</p> Name Type Description Default <code>b</code> <code>bitarray</code> <p>Array of header bits</p> required <p>Returns:</p> Type Description <code>Tuple[int, bitarray]</code> <p>A tuple of the integer value of first varnible and the remaining bits.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_varnibble(b):\n# type: (bitarray) -&gt; Tuple[int, bitarray]\n\"\"\"Reads first varnibble, returns its integer value and remaining bits.\n    :param bitarray b: Array of header bits\n    :return: A tuple of the integer value of first varnible and the remaining bits.\n    :rtype: Tuple[int, bitarray]\n    \"\"\"\nbits = len(b)\nif bits &gt;= 4 and b[0] == 0:\nreturn ba2int(b[:4]), b[4:]\nif bits &gt;= 8 and b[1] == 0:\nreturn ba2int(b[2:8]) + 8, b[8:]\nif bits &gt;= 12 and b[2] == 0:\nreturn ba2int(b[3:12]) + 72, b[12:]\nif bits &gt;= 16 and b[3] == 0:\nreturn ba2int(b[4:16]) + 584, b[16:]\nraise ValueError(\"Invalid bitarray\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_units","title":"<code>encode_units(units)</code>","text":"<p>Encodes a combination of ISCC units to an integer between 0-7 to be used as length value for the final encoding of MT.ISCC</p> <p>Parameters:</p> Name Type Description Default <code>units</code> <code>Tuple</code> <p>A tuple of a MainType combination (can be empty)</p> required <p>Returns:</p> Type Description <code>int</code> <p>Integer value to be used as length-value for header encoding</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_units(units):\n# type: (Tuple[MT, ...]) -&gt; int\n\"\"\"\n    Encodes a combination of ISCC units to an integer between 0-7 to be used as length\n    value for the final encoding of MT.ISCC\n    :param Tuple units: A tuple of a MainType combination (can be empty)\n    :return: Integer value to be used as length-value for header encoding\n    :rtype: int\n    \"\"\"\nreturn UNITS.index(units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_units","title":"<code>decode_units(unit_id)</code>","text":"<p>Decodes an ISCC header length value that has been encoded with a unit_id to an ordered tuple of MainTypes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_units(unit_id):\n# type: (int) -&gt; Tuple[MT, ...]\n\"\"\"\n    Decodes an ISCC header length value that has been encoded with a unit_id to an\n    ordered tuple of MainTypes.\n    \"\"\"\nunits = sorted(UNITS[unit_id])\nreturn tuple(MT(u) for u in units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_length","title":"<code>encode_length(mtype, length)</code>","text":"<p>Encode length to integer value for header encoding.</p> <p>The <code>length</code> value has MainType-specific semantics:</p> <p>For MainTypes <code>META</code>, <code>SEMANTIC</code>, <code>CONTENT</code>, <code>DATA</code>, <code>INSTANCE</code>:</p> <pre><code>Length means number of bits for the body.\nLength is encoded as the multiple of 32-bit chunks (0 being 32bits)\nExamples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n</code></pre> <p>For MainType <code>ISCC</code>:</p> <pre><code>MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\noptional. Length means the composition of optional 64-bit units included\nin the ISCC composite.\n\nExamples:\n    No optional units      -&gt; 0000 -&gt; 0\n    CONTENT                -&gt; 0001 -&gt; 1\n    SEMANTIC               -&gt; 0010 -&gt; 2\n    SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n    META                   -&gt; 0100 -&gt; 4\n    META, CONTENT          -&gt; 0101 -&gt; 5\n    ...\n</code></pre> <p>For MainType <code>ID</code>:</p> <pre><code>Lengths means number the number of bits for the body including the counter\nLength is encoded as number of bytes of the counter (64-bit body is implicit)\nExamples:\n    64 -&gt; 0 (No counter)\n    72 -&gt; 1 (One byte counter)\n    80 -&gt; 2 (Two byte counter)\n    ...\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>The MainType for which to encode the length value.</p> required <code>length</code> <code>Length</code> <p>The length expressed according to the semantics of the type</p> required <p>Returns:</p> Type Description <code>int</code> <p>The length value encoded as integer for use with write_header.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_length(mtype, length):\n# type: (MainType, Length) -&gt; int\n\"\"\"\n    Encode length to integer value for header encoding.\n    The `length` value has MainType-specific semantics:\n    For MainTypes `META`, `SEMANTIC`, `CONTENT`, `DATA`, `INSTANCE`:\n        Length means number of bits for the body.\n        Length is encoded as the multiple of 32-bit chunks (0 being 32bits)\n        Examples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n    For MainType `ISCC`:\n        MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\n        optional. Length means the composition of optional 64-bit units included\n        in the ISCC composite.\n        Examples:\n            No optional units      -&gt; 0000 -&gt; 0\n            CONTENT                -&gt; 0001 -&gt; 1\n            SEMANTIC               -&gt; 0010 -&gt; 2\n            SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n            META                   -&gt; 0100 -&gt; 4\n            META, CONTENT          -&gt; 0101 -&gt; 5\n            ...\n    For MainType `ID`:\n        Lengths means number the number of bits for the body including the counter\n        Length is encoded as number of bytes of the counter (64-bit body is implicit)\n        Examples:\n            64 -&gt; 0 (No counter)\n            72 -&gt; 1 (One byte counter)\n            80 -&gt; 2 (Two byte counter)\n            ...\n    :param MainType mtype: The MainType for which to encode the length value.\n    :param Length length: The length expressed according to the semantics of the type\n    :return: The length value encoded as integer for use with write_header.\n    :rtype: int\n    \"\"\"\nerror = f\"Invalid length {length} for MainType {mtype}\"\n# standard case (length field denotes number of 32-bit chunks, 0 being 32-bits)\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nif length &gt;= 32 and not length % 32:\nreturn (length // 32) - 1\nraise ValueError(error)\n# flag type encoding of included components (pass through as encoded out-of-band)\nelif mtype == MT.ISCC:\nif 0 &lt;= length &lt;= 7:\nreturn length\nraise ValueError(error)\n# counter byte lenght encoding\nelif mtype == MT.ID:\nif 64 &lt;= length &lt;= 96:\nreturn (length - 64) // 8\nraise ValueError(error)\nelse:\nraise ValueError(error)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_length","title":"<code>decode_length(mtype, length)</code>","text":"<p>Dedoce raw length value from ISCC header to length of digest in number of bits.</p> <p>Decodes a raw header integer value in to its semantically meaningfull value (e.g. number of bits)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_length(mtype, length):\n# type: (MainType, Length) -&gt; LN\n\"\"\"\n    Dedoce raw length value from ISCC header to length of digest in number of bits.\n    Decodes a raw header integer value in to its semantically meaningfull value (e.g.\n    number of bits)\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nreturn LN((length + 1) * 32)\nelif mtype == MT.ISCC:\nreturn LN(len(decode_units(length)) * 64 + 128)\nelif mtype == MT.ID:\nreturn LN(length * 8 + 64)\nelse:\nraise ValueError(f\"Invalid length {length} for MainType {mtype}\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32","title":"<code>encode_base32(data)</code>","text":"<p>Standard RFC4648 base32 encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base32 encoding without padding.\n    \"\"\"\nreturn b32encode(data).decode(\"ascii\").rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32","title":"<code>decode_base32(code)</code>","text":"<p>Standard RFC4648 base32 decoding without padding and with casefolding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base32 decoding without padding and with casefolding.\n    \"\"\"\n# python stdlib does not support base32 without padding, so we have to re-pad.\ncl = len(code)\npad_length = math.ceil(cl / 8) * 8 - cl\nreturn bytes(b32decode(code + \"=\" * pad_length, casefold=True))\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_decompose","title":"<code>iscc_decompose(iscc_code)</code>","text":"<p>Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.</p> <p>A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated by a hyphen.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decompose(iscc_code):\n# type: (str) -&gt; List[str]\n\"\"\"\n    Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.\n    A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated\n    by a hyphen.\n    \"\"\"\niscc_code = iscc_clean(iscc_code)\ncomponents = []\nraw_code = decode_base32(iscc_code)\nwhile raw_code:\nmt, st, vs, ln, body = decode_header(raw_code)\n# standard ISCC-UNIT with tail continuation\nif mt != MT.ISCC:\nln_bits = decode_length(mt, ln)\ncode = encode_component(mt, st, vs, ln_bits, body[: ln_bits // 8])\ncomponents.append(code)\nraw_code = body[ln_bits // 8 :]\ncontinue\n# ISCC-CODE\nmain_types = decode_units(ln)\n# rebuild dynamic units (META, SEMANTIC, CONTENT)\nfor idx, mtype in enumerate(main_types):\nstype = ST.NONE if mtype == MT.META else st\ncode = encode_component(mtype, stype, vs, 64, body[idx * 8 :])\ncomponents.append(code)\n# rebuild static units (DATA, INSTANCE)\ndata_code = encode_component(MT.DATA, ST.NONE, vs, 64, body[-16:-8])\ninstance_code = encode_component(MT.INSTANCE, ST.NONE, vs, 64, body[-8:])\ncomponents.extend([data_code, instance_code])\nbreak\nreturn components\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_normalize","title":"<code>iscc_normalize(iscc_code)</code>","text":"<p>Normalize an ISCC to its canonical form.</p> <p>The canonical form of an ISCC is its shortest base32 encoded representation prefixed with the string <code>ISCC:</code>.</p> <p>Possible valid inputs:</p> <pre><code>MEACB7X7777574L6\nISCC:MEACB7X7777574L6\nfcc010001657fe7cafe9791bb\niscc:maagztfqttvizpjr\nIscc:Maagztfqttvizpjr\n</code></pre> <p>Info</p> <p>A concatenated sequence of codes will be composed into a single ISCC of MainType <code>MT.ISCC</code> if possible.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core\n&gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Normalized ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_normalize(iscc_code):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize an ISCC to its canonical form.\n    The canonical form of an ISCC is its shortest base32 encoded representation\n    prefixed with the string `ISCC:`.\n    Possible valid inputs:\n        MEACB7X7777574L6\n        ISCC:MEACB7X7777574L6\n        fcc010001657fe7cafe9791bb\n        iscc:maagztfqttvizpjr\n        Iscc:Maagztfqttvizpjr\n    !!! info\n        A concatenated sequence of codes will be composed into a single ISCC of MainType\n        `MT.ISCC` if possible.\n    !!! example\n        ``` py\n        &gt;&gt;&gt; import iscc_core\n        &gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n        'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n        ```\n    :param str iscc_code: Any valid ISCC string\n    :return: Normalized ISCC\n    :rtype: str\n    \"\"\"\nfrom iscc_core.iscc_code import gen_iscc_code_v0\ndecoders = {\nMULTIBASE.base16.value: bytes.fromhex,  # f\nMULTIBASE.base32.value: decode_base32,  # b\nMULTIBASE.base32hex.value: decode_base32hex,  # v\nMULTIBASE.base58btc.value: base58.b58decode,  # z\nMULTIBASE.base64url.value: decode_base64,  # u\n}\n# Transcode to base32 if &lt;multibase&gt;&lt;multicodec&gt; encoded\nmultibase_prefix = iscc_code[0]\nif multibase_prefix in decoders.keys():\ndecoder = decoders[multibase_prefix]\ndecoded = decoder(iscc_code[1:])\nif not decoded.startswith(MC_PREFIX):\nraise ValueError(f\"Malformed multiformat codec: {decoded[:2]}\")\niscc_code = encode_base32(decoded[2:])\nelse:\nprefix = iscc_code.lstrip(\"ISCC:\").lstrip(\"iscc:\")[:2].upper()\nif prefix not in PREFIXES:\nraise ValueError(f\"ISCC starts with invalid prefix {prefix}\")\ndecomposed = iscc_decompose(iscc_code)\nrecomposed = gen_iscc_code_v0(decomposed)[\"iscc\"] if len(decomposed) &gt;= 2 else decomposed[0]\nreturn f\"ISCC:{recomposed}\" if not recomposed.startswith(\"ISCC:\") else recomposed\n</code></pre>"},{"location":"codec/#alternate-encodings","title":"Alternate Encodings","text":""},{"location":"codec/#iscc_core.codec.encode_base64","title":"<code>encode_base64(data)</code>","text":"<p>Standard RFC4648 base64url encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base64(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base64url encoding without padding.\n    \"\"\"\ncode = urlsafe_b64encode(data).decode(\"ascii\")\nreturn code.rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base64","title":"<code>decode_base64(code)</code>","text":"<p>Standard RFC4648 base64url decoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base64(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base64url decoding without padding.\n    \"\"\"\npadding = 4 - (len(code) % 4)\nstring = code + (\"=\" * padding)\nreturn urlsafe_b64decode(string)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32hex","title":"<code>encode_base32hex(data)</code>","text":"<p>RFC4648 Base32hex encoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32hex(data):\n# type: (bytes) -&gt;  str\n\"\"\"\n    RFC4648 Base32hex encoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = encode_base32(data)\nreturn b32.translate(b32_to_hex)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32hex","title":"<code>decode_base32hex(code)</code>","text":"<p>RFC4648 Base32hex decoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32hex(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    RFC4648 Base32hex decoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = code.translate(hex_to_b32)\nreturn decode_base32(b32)\n</code></pre>"},{"location":"codec/#helper-functions","title":"Helper Functions","text":""},{"location":"codec/#iscc_core.codec.iscc_decode","title":"<code>iscc_decode(iscc)</code>","text":"<p>Decode ISCC to an IsccTuple</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>ISCC decoded to a tuple</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decode(iscc):\n# type: (str) -&gt; IsccTuple\n\"\"\"\n    Decode ISCC to an IsccTuple\n    :param str iscc: ISCC string\n    :return: ISCC decoded to a tuple\n    :rtype: IsccTuple\n    \"\"\"\niscc = iscc_clean(iscc_normalize(iscc))\ndata = decode_base32(iscc)\nreturn decode_header(data)\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_explain","title":"<code>iscc_explain(iscc)</code>","text":"<p>Convert ISCC to a human-readable representation</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Human-readable representation of ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_explain(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Convert ISCC to a human-readable representation\n    :param str iscc: ISCC string\n    :return: Human-readable representation of ISCC\n    :rtype: str\n    \"\"\"\ntid = iscc_type_id(iscc)\nfields = iscc_decode(iscc)\nif fields[0] == MT.ID:\ncounter_bytes = fields[-1][8:]\nif counter_bytes:\ncounter = uvarint.decode(counter_bytes)\nhex_hash = fields[-1][:8].hex()\nreturn f\"{tid}-{hex_hash}-{counter.integer}\"\nhex_hash = fields[-1].hex()\nreturn f\"{tid}-{hex_hash}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_type_id","title":"<code>iscc_type_id(iscc)</code>","text":"<p>Extract and convert ISCC HEADER to a readable Type-ID string.</p> <p>Type-ids can be used as names in databases to index ISCC-UNITs seperatly.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Unique Type-ID string</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_type_id(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Extract and convert ISCC HEADER to a readable Type-ID string.\n    Type-ids can be used as names in databases to index ISCC-UNITs seperatly.\n    :param str iscc: ISCC string\n    :return: Unique Type-ID string\n    :rtype: str\n    \"\"\"\nfields = iscc_decode(iscc)\nmtype = MT(fields[0])\nstype = SUBTYPE_MAP[fields[0]](fields[1])\nif mtype == MT.ISCC:\nmtypes = decode_units(fields[3])\nlength = \"\".join([t.name[0] for t in mtypes]) + \"DI\"\nelse:\nlength = decode_length(fields[0], fields[3])\nversion = VS(fields[2])\nreturn f\"{mtype.name}-{stype.name}-{version.name}-{length}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_validate","title":"<code>iscc_validate(iscc, strict = True)</code>","text":"<p>Validate that a given string is a strictly well-formed ISCC.</p> <p>A strictly well-formed ISCC is:</p> <ul> <li>an ISCC-CODE or ISCC-UNIT</li> <li>encoded with base32 upper without padding</li> <li>has a valid combination of header values</li> <li>is represented in its canonical URI form</li> </ul> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <code>strict</code> <code>bool</code> <p>Raise an exeption if validation fails (default True)</p> <code>True</code> <p>Returns:</p> Type Description <code>bool</code> <p>True if sting is valid else false. (raises ValueError in strict mode)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_validate(iscc, strict=True):\n# type: (str, bool) -&gt; bool\n\"\"\"\n    Validate that a given string is a *strictly well-formed* ISCC.\n    A *strictly well-formed* ISCC is:\n    - an ISCC-CODE or ISCC-UNIT\n    - encoded with base32 upper without padding\n    - has a valid combination of header values\n    - is represented in its canonical URI form\n    :param str iscc: ISCC string\n    :param bool strict: Raise an exeption if validation fails (default True)\n    :return: True if sting is valid else false. (raises ValueError in strict mode)\n    :rtype: bool\n    \"\"\"\n# Basic regex validation\nmatch = re.match(\"^ISCC:[A-Z2-7]{10,60}$\", iscc)\nif not match:\nif strict:\nraise ValueError(\"ISCC string does not match ^ISCC:[A-Z2-7]{10,60}$\")\nelse:\nreturn False\ncleaned = iscc_clean(iscc)\nprefix = cleaned[:2]\nif prefix not in PREFIXES:\nif strict:\nraise ValueError(f\"Header starts with invalid sequence {prefix}\")\nelse:\nreturn False\nm, s, v, l, t = decode_header(decode_base32(cleaned))\nif v != 0:\nif strict:\nraise ValueError(f\"Unknown version {v} in version header\")\nelse:\nreturn False\nreturn True\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_clean","title":"<code>iscc_clean(iscc)</code>","text":"<p>Cleanup ISCC string.</p> <p>Removes leading scheme, dashes, leading/trailing whitespace.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Cleaned ISCC string.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_clean(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Cleanup ISCC string.\n    Removes leading scheme, dashes, leading/trailing whitespace.\n    :param str iscc: Any valid ISCC string\n    :return: Cleaned ISCC string.\n    :rtype: str\n    \"\"\"\nsplit = [part.strip() for part in iscc.strip().split(\":\")]\nif len(split) == 1:\ncode = split[0]\n# remove dashes if not multiformat\nif code[0] not in list(MULTIBASE):\ncode = code.replace(\"-\", \"\")\nreturn code\nelif len(split) == 2:\nscheme, code = split\nif scheme.lower() != \"iscc\":\nraise ValueError(f\"Invalid scheme: {scheme}\")\nreturn code.replace(\"-\", \"\")\nelse:\nraise ValueError(f\"Malformed ISCC string: {iscc}\")\n</code></pre>"},{"location":"options/options/","title":"ISCC-CORE - Configuration Options","text":"<p>Options for the iscc-core package can be configured using environment variables. Variables are loaded as class-attributes on the <code>CoreOptions</code> instance. Environment variables are named like the class-attribute but prefixed with <code>ISCC_CORE_</code> and upper-cased.</p> <p>Example how to access configuration options</p> <pre><code>import iscc_core as ic\n# To access ISCC_CORE_TEXT_NGRAM_SIZE setting use\ntext_ngram_size: int = ic.core_opts.text_ngram_size\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions","title":"CoreOptions","text":"<p>Parameters with defaults for ISCC calculations.</p>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_bits","title":"meta_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>meta_bits: int = Field(\n64,\ndescription=\"Default length of generated Meta-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_name","title":"meta_trim_name  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>meta_trim_name: int = Field(\n128, description=\"Trim `name` to this mumber of bytes\"\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_description","title":"meta_trim_description  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>meta_trim_description: int = Field(\n4096,\ndescription=\"Trim `description` to this number of bytes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_text","title":"meta_ngram_size_text  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>meta_ngram_size_text: int = Field(\n3,\ndescription=\"Sliding window width (characters) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_bytes","title":"meta_ngram_size_bytes  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>meta_ngram_size_bytes: int = Field(\n4,\ndescription=\"Sliding window width (bytes) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_bits","title":"text_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>text_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Text in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_ngram_size","title":"text_ngram_size  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>text_ngram_size: int = Field(\n13,\ndescription=\"Number of characters per feature hash (size of sliding window)\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_unicode_filter","title":"text_unicode_filter  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>text_unicode_filter: frozenset = Field(\nfrozenset({\"C\", \"M\", \"P\"}),\ndescription=\"Unicode categories to remove during text normalization\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_newlines","title":"text_newlines  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>text_newlines: frozenset = Field(\nfrozenset(\n{\n\"\\n\",\n\"\\x0b\",\n\"\\x0c\",\n\"\\r\",\n\"\\x85\",\n\"\\u2028\",\n\"\\u2029\",\n}\n),\ndescription=\"Characters regarded as newline characters for normalization purposes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.image_bits","title":"image_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>image_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Image in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.audio_bits","title":"audio_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>audio_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Audio in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.video_bits","title":"video_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>video_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Video in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_bits","title":"data_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>data_bits: int = Field(\n64,\ndescription=\"Default length of generated Data-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_avg_chunk_size","title":"data_avg_chunk_size  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>data_avg_chunk_size: int = Field(\n1024,\ndescription=\"Target chunk size for data chunking in number of bytes.\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.instance_bits","title":"instance_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>instance_bits: int = Field(\n64,\ndescription=\"Default length of generated Instance-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.mixed_bits","title":"mixed_bits  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>mixed_bits: int = Field(\n64,\ndescription=\"Default length of generated Mixed-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.io_read_size","title":"io_read_size  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>io_read_size: int = Field(\n2097152,\ndescription=\"File read buffer size in bytes for hashing operations\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.cdc_gear","title":"cdc_gear  <code>class-attribute</code> <code>instance-attribute</code>","text":"<pre><code>cdc_gear: Tuple = Field(\n(\n1553318008,\n574654857,\n759734804,\n310648967,\n1393527547,\n1195718329,\n694400241,\n1154184075,\n1319583805,\n1298164590,\n122602963,\n989043992,\n1918895050,\n933636724,\n1369634190,\n1963341198,\n1565176104,\n1296753019,\n1105746212,\n1191982839,\n1195494369,\n29065008,\n1635524067,\n722221599,\n1355059059,\n564669751,\n1620421856,\n1100048288,\n1018120624,\n1087284781,\n1723604070,\n1415454125,\n737834957,\n1854265892,\n1605418437,\n1697446953,\n973791659,\n674750707,\n1669838606,\n320299026,\n1130545851,\n1725494449,\n939321396,\n748475270,\n554975894,\n1651665064,\n1695413559,\n671470969,\n992078781,\n1935142196,\n1062778243,\n1901125066,\n1935811166,\n1644847216,\n744420649,\n2068980838,\n1988851904,\n1263854878,\n1979320293,\n111370182,\n817303588,\n478553825,\n694867320,\n685227566,\n345022554,\n2095989693,\n1770739427,\n165413158,\n1322704750,\n46251975,\n710520147,\n700507188,\n2104251000,\n1350123687,\n1593227923,\n1756802846,\n1179873910,\n1629210470,\n358373501,\n807118919,\n751426983,\n172199468,\n174707988,\n1951167187,\n1328704411,\n2129871494,\n1242495143,\n1793093310,\n1721521010,\n306195915,\n1609230749,\n1992815783,\n1790818204,\n234528824,\n551692332,\n1930351755,\n110996527,\n378457918,\n638641695,\n743517326,\n368806918,\n1583529078,\n1767199029,\n182158924,\n1114175764,\n882553770,\n552467890,\n1366456705,\n934589400,\n1574008098,\n1798094820,\n1548210079,\n821697741,\n601807702,\n332526858,\n1693310695,\n136360183,\n1189114632,\n506273277,\n397438002,\n620771032,\n676183860,\n1747529440,\n909035644,\n142389739,\n1991534368,\n272707803,\n1905681287,\n1210958911,\n596176677,\n1380009185,\n1153270606,\n1150188963,\n1067903737,\n1020928348,\n978324723,\n962376754,\n1368724127,\n1133797255,\n1367747748,\n1458212849,\n537933020,\n1295159285,\n2104731913,\n1647629177,\n1691336604,\n922114202,\n170715530,\n1608833393,\n62657989,\n1140989235,\n381784875,\n928003604,\n449509021,\n1057208185,\n1239816707,\n525522922,\n476962140,\n102897870,\n132620570,\n419788154,\n2095057491,\n1240747817,\n1271689397,\n973007445,\n1380110056,\n1021668229,\n12064370,\n1186917580,\n1017163094,\n597085928,\n2018803520,\n1795688603,\n1722115921,\n2015264326,\n506263638,\n1002517905,\n1229603330,\n1376031959,\n763839898,\n1970623926,\n1109937345,\n524780807,\n1976131071,\n905940439,\n1313298413,\n772929676,\n1578848328,\n1108240025,\n577439381,\n1293318580,\n1512203375,\n371003697,\n308046041,\n320070446,\n1252546340,\n568098497,\n1341794814,\n1922466690,\n480833267,\n1060838440,\n969079660,\n1836468543,\n2049091118,\n2023431210,\n383830867,\n2112679659,\n231203270,\n1551220541,\n1377927987,\n275637462,\n2110145570,\n1700335604,\n738389040,\n1688841319,\n1506456297,\n1243730675,\n258043479,\n599084776,\n41093802,\n792486733,\n1897397356,\n28077829,\n1520357900,\n361516586,\n1119263216,\n209458355,\n45979201,\n363681532,\n477245280,\n2107748241,\n601938891,\n244572459,\n1689418013,\n1141711990,\n1485744349,\n1181066840,\n1950794776,\n410494836,\n1445347454,\n2137242950,\n852679640,\n1014566730,\n1999335993,\n1871390758,\n1736439305,\n231222289,\n603972436,\n783045542,\n370384393,\n184356284,\n709706295,\n1453549767,\n591603172,\n768512391,\n854125182,\n),\ndescription=\"Random gear vector\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformanc_critical","title":"conformanc_critical  <code>module-attribute</code>","text":"<pre><code>conformanc_critical = {\n\"meta_trim_name\",\n\"meta_trim_description\",\n\"meta_ngram_size_text\",\n\"meta_ngram_size_bytes\",\n\"text_ngram_size\",\n\"text_unicode_filter\",\n\"text_newlines\",\n\"data_avg_chunk_size\",\n\"cdc_gear\",\n}\n</code></pre>"},{"location":"options/options/#iscc_core.options.has_logged_confromance","title":"has_logged_confromance  <code>module-attribute</code>","text":"<pre><code>has_logged_confromance = False\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformance_check_options","title":"conformance_check_options","text":"<pre><code>conformance_check_options(opts)\n</code></pre> <p>Check and log if options have non-default conformance critical values</p>"},{"location":"options/options/#iscc_core.options.core_opts","title":"core_opts  <code>module-attribute</code>","text":"<pre><code>core_opts = CoreOptions()\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformant_options","title":"conformant_options  <code>module-attribute</code>","text":"<pre><code>conformant_options = conformance_check_options(core_opts)\n</code></pre>"},{"location":"units/","title":"ISCC - UNITs","text":"<p>A standard ISCC-CODE is build from multiple ISCC-UNITs. Each unit serve a different purpose.</p>"},{"location":"units/code_data/","title":"ISCC - Data-Code","text":"<p>A similarity perserving hash for binary data (soft hash).</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code","title":"<code>gen_data_code(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create a similarity preserving ISCC Data-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code_v0","title":"<code>gen_data_code_v0(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.soft_hash_data_v0","title":"<code>soft_hash_data_v0(stream)</code>","text":"<p>Create a similarity preserving Data-Hash digest</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Data-Hash (soft-hash) digest used as body for Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0","title":"<code>DataHasherV0</code>","text":"<p>Incremental Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.__init__","title":"<code>__init__(data = None)</code>","text":"<p>Create a DataHasher</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Optional[Data]</code> <p>initial payload for hashing.</p> <code>None</code>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.digest","title":"<code>digest()</code>","text":"<p>Calculate 256-bit minhash digest from feature hashes.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.code","title":"<code>code(bits = ic.core_opts.data_bits)</code>","text":"<p>Encode digest as an ISCC Data-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Data-Code</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Data-Code</p>"},{"location":"units/code_flake/","title":"ISCC - Flake-Code","text":"<p>A unique, time-sorted identifier composed of an 48-bit timestamp and 16 to 208 bit randomness.</p> <p>The ISCC Flake-Code is a unique identifier for distributed ID generation. The 64-bit version can be used as efficient surrogate key in database systems. It has guaranteed uniqueness if generated from a singele process and is time sortable in integer and base32hex representation. The 128-bit version is a K-sortable, globally unique identifier for use in distributed systems and is compatible with UUID.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core as ic\n&gt;&gt;&gt; ic.gen_flake_code(bits=64)\n{'iscc': 'ISCC:OAAQC7YN7PG2XOR4'}\n&gt;&gt;&gt; ic.gen_flake_code(bits=128)\n{'iscc': 'ISCC:OABQC7YN7RJGUUTLKDSKBXO25MA5E'}\n# Or use the convenience Flake class for easy access to different representations\n&gt;&gt;&gt; flake = ic.Flake(bits=64)\n&gt;&gt;&gt; flake.iscc\n'ISCC:OAAQC7YOADBZYNF7'\n&gt;&gt;&gt; flake.time\n'2022-02-18T18:03:25.468'\n&gt;&gt;&gt; flake.int\n107820312524764351\n&gt;&gt;&gt; flake.string\n'05VGS063JGQBU'\n</code></pre>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code","title":"<code>gen_flake_code(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest standard algorithm</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code_v0","title":"<code>gen_flake_code_v0(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest algorithm v0</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.uid_flake_v0","title":"<code>uid_flake_v0(ts = None, bits = ic.core_opts.flake_bits)</code>","text":"<p>Generate time and randomness based Flake-Hash</p> <p>Parameters:</p> Name Type Description Default <code>ts</code> <code>Optional[float]</code> <p>Unix timestamp (defaults to current time)</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length resulting Flake-Code (multiple of 32)</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Flake-Hash digest</p>"},{"location":"units/code_instance/","title":"ISCC - Instance-Code","text":"<p>A data checksum.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code","title":"<code>gen_instance_code(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with properties: iscc, datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code_v0","title":"<code>gen_instance_code_v0(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.hash_instance_v0","title":"<code>hash_instance_v0(stream)</code>","text":"<p>Create 256-bit hash digest for the Instance-Code body</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for hash generation.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Instance-Hash digest used as body of Instance-Code</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0","title":"<code>InstanceHasherV0</code>","text":"<p>Incremental Instance-Hash generator.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Instance-Hash generator.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Data</code> <p>Data to be hashed</p> required"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.digest","title":"<code>digest()</code>","text":"<p>Return Instance-Hash</p> <p>Returns:</p> Type Description <code>bytes</code> <p>Instance-Hash digest</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.multihash","title":"<code>multihash()</code>","text":"<p>Return blake3 multihash</p> <p>Returns:</p> Type Description <code>str</code> <p>Blake3 hash as 256-bit multihash</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.code","title":"<code>code(bits = ic.core_opts.instance_bits)</code>","text":"<p>Encode digest as an ISCC Instance-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Instance-Code</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Instance-Code</p>"},{"location":"units/code_meta/","title":"ISCC - Meta-Code","text":"<p>A similarity preserving hash for digital asset metadata.</p>"},{"location":"units/code_meta/#purpose","title":"Purpose","text":"<p>The Meta-Code is the first possible (optional) unit of an ISCC-CODE. It is calculated from the metadata of a digital asset. The primary purpose of the Meta-Code is to aid the discovery of digital assets with similar metadata and the detection of metadata anomalies. As a secondary function, Meta-Code processing also creates a secure Meta-Hash for cryptogrpahic binding purposes.</p>"},{"location":"units/code_meta/#inputs","title":"Inputs","text":"<p>The metadata supplied for Meta-Code calculation is called Seed-Metadata. Seed-Metadata has 3 possible elements:</p> <ul> <li>name (required): The name or title of the work manifested by the digital asset.</li> <li>description (optional): A disambiguating textual description of the digital asset.</li> <li>meta (optional): Industry-sector or use-case specific metadata, encoded as     Data-URL (RFC-2397).</li> </ul> <p>Note</p> <p>Due to the broad applicability of the ISCC we do not prescribe a particular schema for the <code>meta</code>-element. Instead we use the Data-URL format because it can encode and self-describe any conceivable metadata in a sufficently machine-interpretable form at any desired specificity.</p> <p>Data-URL Examples:</p> <ul> <li>Metadata is \"some\" JSON: <code>data:application/json;base64,&lt;data&gt;</code></li> <li>Metadata is JSON-LD: <code>data:application/ld+json;base64,&lt;data&gt;</code></li> <li>Metadata is \"some\" XML: <code>data:application/xml;base64,&lt;data&gt;</code></li> <li>Metadata is MARC21 XML: <code>data:application/marcxml+xml;base64,&lt;data&gt;</code></li> <li>Metadata is IPTC NewsML: <code>data:application/vnd.iptc.g2.newsitem+xml;base64,&lt;data&gt;</code></li> </ul> <p>Data-URLs are also supported by all major internet browsers.</p>"},{"location":"units/code_meta/#processing","title":"Processing","text":""},{"location":"units/code_meta/#meta-code","title":"Meta-Code","text":"<p>The first 32-bits of a Meta-Code are calculated as a simliarity hash from the <code>name</code> field. The second 32-bits are also calculated from the <code>name</code> field if no other input was supplied. If <code>description</code> is suplied but no <code>meta</code>, the <code>description</code> will be used for the second 32-bits. If <code>meta</code> is supplied it will be used in favour of <code>description</code> for the second 32-bits.</p> <pre><code>flowchart LR\n    B{name?}\n    B --&gt;|Yes| J[\"P1 = SH(name)\"]\n    J --&gt; D{meta?}\n    D --&gt;|Yes| F[\"Meta-Code = P1 + SH(meta)\"]\n    D --&gt;|No| G{description?}\n    G --&gt;|Yes| H[\"Meta-Code = P1 + SH(description)\"]\n    G --&gt;|No| I[\"Meta-Code = P1\"]\n    B --&gt;|No| E[Skip Meta-Code]</code></pre> <p>Note</p> <p>To support automation and reproducibility, applications that generate ISCCs, should prioritize metadata that is automatically extracted from the digital asset.</p> <p>If embedded metadata is not available or known to be unreliable an application should rely on external metadata or explicitly ask users to supply at least the <code>name</code>-field. Applications should then first embed metadata into the asset before calculating the ISCC-CODE. This ensures that the embedded metadata is bound to the asset by the Instance-Code.</p> <p>If neither embedded nor external metadata is available, the application may resort to use the filename of the digital asset as value for the <code>name</code>-field. If no value can be determined for the <code>name</code>-field, an application shall skip generation of a Meta-Code and create an ISCC-CODE without a Meta-Code.</p>"},{"location":"units/code_meta/#meta-hash","title":"Meta-Hash","text":"<p>In addition to the Meta-Code we also create a cryptographic hash (the Meta-Hash) of the supplied Seed-Metadata. It is used to securely bind metadata to the digital asset.</p> <pre><code>flowchart LR\n    N{name?}\n    N --&gt;|Yes| A{meta?}\n    N --&gt;|No| E[Skip Meta-Hash]\n    A --&gt;|Yes| H[\"Meta-Hash = H(meta)\"]\n    A --&gt;|No| B{description?}\n    B --&gt;|Yes| HND[\"Meta-Hash = H(name + description)\"]\n    B --&gt;|No| HN[\"Meta-Hash = H(name)\"]</code></pre>"},{"location":"units/code_meta/#functions","title":"Functions","text":""},{"location":"units/code_meta/#iscc_core.code_meta.gen_meta_code_v0","title":"<code>gen_meta_code_v0(name, description = None, meta = None, bits = ic.core_opts.meta_bits)</code>","text":"<p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.soft_hash_meta_v0","title":"<code>soft_hash_meta_v0(name, extra = None)</code>","text":"<p>Calculate simmilarity preserving 256-bit hash digest from asset metadata.</p> <p>Textual input should be stripped of markup, normalized and trimmed before hashing. Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be serialized in a canonical form (for example JCS for JSON)</p> <p>Note</p> <p>The processing algorithm depends on the type of the <code>extra</code> input. If the <code>extra</code> field is supplied and non-empty, we create separate hashes for <code>name</code> and <code>extra</code> and interleave them in 32-bit chunks:</p> <ul> <li> <p>If the <code>extra</code> input is <code>None</code> or an empty <code>str</code>/<code>bytes</code> object the Meta-Hash will be generated from the <code>name</code>-field only.</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty text string (str) the string is lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty bytes object the processing is done bytewise.</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Title of the work manifested in the digital asset</p> required <code>extra</code> <code>Union[str,bytes,None]</code> <p>Additional metadata for disambiguation</p> <code>None</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit simhash digest for Meta-Code</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def soft_hash_meta_v0(name, extra=None):\n# type: (str, Union[str,bytes,None]) -&gt; bytes\n\"\"\"\n    Calculate simmilarity preserving 256-bit hash digest from asset metadata.\n    Textual input should be stripped of markup, normalized and trimmed before hashing.\n    Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be\n    serialized in a canonical form (for example\n    [JCS](https://tools.ietf.org/id/draft-rundgren-json-canonicalization-scheme-00.html) for JSON)\n    !!! note\n        The processing algorithm depends on the type of the `extra` input.\n        If the `extra` field is supplied and non-empty, we create separate hashes for\n        `name` and `extra` and interleave them in 32-bit chunks:\n        - If the `extra` input is `None` or an empty `str`/`bytes` object the Meta-Hash will\n        be generated from the `name`-field only.\n        - If the `extra`-input is a non-empty **text** string (str) the string is\n        lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).\n        - If the `extra`-input is a non-empty **bytes** object the processing is done bytewise.\n    :param str name: Title of the work manifested in the digital asset\n    :param Union[str,bytes,None] extra: Additional metadata for disambiguation\n    :return: 256-bit simhash digest for Meta-Code\n    :rtype: bytes\n    \"\"\"\nname = ic.text_collapse(name)\nname_n_grams = ic.sliding_window(name, width=ic.core_opts.meta_ngram_size_text)\nname_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in name_n_grams]\nsimhash_digest = ic.alg_simhash(name_hash_digests)\nif extra in {None, \"\", b\"\"}:\nreturn simhash_digest\nelse:\n# Augment with interleaved hash for extra metadata\nif isinstance(extra, bytes):\n# Raw bytes are handled per byte\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_bytes)\nextra_hash_digests = [blake3(ngram).digest() for ngram in extra_n_grams]\nelif isinstance(extra, str):\n# Text is collapsed and handled per character (multibyte)\nextra = ic.text_collapse(extra)\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_text)\nextra_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in extra_n_grams]\nelse:\nraise ValueError(\"parameter `extra` must be of type str or bytes!\")\nextra_simhash_digest = ic.alg_simhash(extra_hash_digests)\n# Interleave first half of name and extra simhashes in 32-bit chunks\nchunks_simhash_digest = sliced(simhash_digest[:16], 4)\nchunks_extra_simhash_digest = sliced(extra_simhash_digest[:16], 4)\ninterleaved = interleave(chunks_simhash_digest, chunks_extra_simhash_digest)\nsimhash_digest = bytearray()\nfor chunk in interleaved:\nsimhash_digest += chunk\nsimhash_digest = bytes(simhash_digest)\nreturn simhash_digest\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_clean","title":"<code>text_clean(text)</code>","text":"<p>Clean text for display.</p> <ul> <li>Normalize with NFKC normalization.</li> <li>Remove Control Characters (except newlines)</li> <li>Reduce multiple consecutive newlines to a maximum of two newlines</li> <li>Strip leading and trailing whitespace</li> </ul> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_clean(text):\n# type: (str) -&gt; str\n\"\"\"\n    Clean text for display.\n    - Normalize with NFKC normalization.\n    - Remove Control Characters (except newlines)\n    - Reduce multiple consecutive newlines to a maximum of two newlines\n    - Strip leading and trailing whitespace\n    \"\"\"\n# Unicode normalize\ntext = unicodedata.normalize(\"NFKC\", text)\n# Remove control characters\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] != \"C\" or ch in ic.core_opts.text_newlines\n)\n# Collapse more than two consecutive newlines\nchars = []\nnewline_count = 0\nfor c in text:\nif c in ic.core_opts.text_newlines:\nif newline_count &lt; 2:\nchars.append(\"\\n\")\nnewline_count += 1\ncontinue\nelse:\nnewline_count = 0\nchars.append(c)\ntext = \"\".join(chars)\nreturn text.strip()\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_remove_newlines","title":"<code>text_remove_newlines(text)</code>","text":"<p>Remove newlines.</p> <p>The <code>name</code> field serves as a displayable title. We remove newlines and leading and trailing whitespace. We also collapse consecutive spaces to single spaces.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <p>Text for newline removal</p> required <p>Returns:</p> Type Description <code>str</code> <p>Single line of text</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_remove_newlines(text):\n# type: (str) -&gt; str\n\"\"\"\n    Remove newlines.\n    The `name` field serves as a displayable title. We remove newlines and leading and trailing\n    whitespace. We also collapse consecutive spaces to single spaces.\n    :param text: Text for newline removal\n    :return: Single line of text\n    :rtype: str\n    \"\"\"\nreturn \" \".join(text.split())\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_trim","title":"<code>text_trim(text, nbytes)</code>","text":"<p>Trim text such that its utf-8 encoded size does not exceed <code>nbytes</code>.</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_trim(text, nbytes):\n# type: (str, int) -&gt; str\n\"\"\"Trim text such that its utf-8 encoded size does not exceed `nbytes`.\"\"\"\nreturn text.encode(\"utf-8\")[:nbytes].decode(\"utf-8\", \"ignore\").strip()\n</code></pre>"},{"location":"units/content/","title":"ISCC - Content-Codes","text":""},{"location":"units/content/code_content_audio/","title":"ISCC - Audio-Code","text":"<p>A similarity preserving hash for audio content (soft hash).</p> <p>Creates an ISCC object that provides an <code>iscc</code>-field with an Audio-Code and a <code>duration</code>-field.</p> <p>The Content-Code Audio is generated from a Chromaprint fingerprint provided as a vector of 32-bit signed integers. The iscc-sdk uses fpcalc to extract Chromaprint vectors with the following command line parameters:</p> <p><code>$ fpcalc -raw -json -signed -length 0 myaudiofile.mp3</code></p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code","title":"<code>gen_audio_code(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code_v0","title":"<code>gen_audio_code_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.soft_hash_audio_v0","title":"<code>soft_hash_audio_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create audio similarity hash from a chromaprint vector.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting similarity hash (multiple of 32)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Audio-Hash digest</p>"},{"location":"units/content/code_content_image/","title":"ISCC - Image-Code","text":"<p>A similarity preserving perceptual hash for images.</p> <p>The ISCC Content-Code Image is created by calculating a discrete cosine transform on normalized image-pixels and comparing the values from the upper left area of the dct-matrix against their median values to set the hash-bits.</p> <p>Images must be normalized before using gen_image_code. Prepare images as follows:</p> <ul> <li>Transpose image according to EXIF Orientation</li> <li>Add white background to image if it has alpha transparency</li> <li>Crop empty borders of image</li> <li>Convert image to grayscale</li> <li>Resize image to 32x32</li> <li>Flatten 32x32 matrix to an array of 1024 grayscale (uint8) pixel values</li> </ul>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code","title":"<code>gen_image_code(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values).</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code_v0","title":"<code>gen_image_code_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.soft_hash_image_v0","title":"<code>soft_hash_image_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Calculate image hash from normalized grayscale pixel sequence of length 1024.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> required <code>bits</code> <code>int</code> <p>Bit-length of image hash (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving Image-Hash digest.</p>"},{"location":"units/content/code_content_mixed/","title":"ISCC - Mixed Code","text":"<p>A similarity hash for mixed media content.</p> <p>Creates an ISCC object that provides a <code>iscc</code>-field a Mixed-Code and a <code>parts</code>-field that lists the input codes.</p> <p>Many digital assets embed multiple assets of different mediatypes in a single file. Text documents may include images, video includes audio in most cases. The ISCC Content-Code-Mixed encodes the similarity of a collection of assets of the same or different mediatypes that may occur in a multimedia asset.</p> <p>Applications that create mixed Content-Codes must be capable to extract embedded assets and create individual Content-Codes per asset.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code","title":"<code>gen_mixed_code(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code Mixed with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code_v0","title":"<code>gen_mixed_code_v0(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.soft_hash_codes_v0","title":"<code>soft_hash_codes_v0(cc_digests, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create a similarity hash from multiple Content-Code digests.</p> <p>The similarity hash is created from the bodies of the input codes with the first byte of the code-header prepended.</p> <p>All codes must be of main-type CONTENT and have a minimum length of <code>bits</code>.</p> <p>Parameters:</p> Name Type Description Default <code>cc_digests</code> <code>Sequence[bytes]</code> <p>a list of Content-Code digests.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving byte hash.</p>"},{"location":"units/content/code_content_text/","title":"ISCC - Text Code","text":"<p>A similarity preserving hash for plain-text content (soft hash).</p> <p>The ISCC Text-Code is generated from plain-text that has been extracted from a media assets.</p> <p>Warning</p> <p>Plain-text extraction from documents in various formats (especially PDF) may yield very diffent results depending on the extraction tools being used. The iscc-sdk uses Apache Tika to extract text from documents for Text-Code generation.</p> <p>Algorithm overview</p> <ul> <li>Apply <code>text_collapse</code> function to text input</li> <li>Count characters of collapsed text</li> <li>Apply <code>soft_hash_text_v0</code> to collapsed text</li> </ul>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.gen_text_code_v0","title":"<code>gen_text_code_v0(text, bits = ic.core_opts.text_bits)</code>","text":"<p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.text_collapse","title":"<code>text_collapse(text)</code>","text":"<p>Normalize and simplify text for similarity hashing.</p> <ul> <li>Decompose with NFD normalization.</li> <li>Remove all whitespace characters and convert text to lower case</li> <li>Filter control characters, marks (diacritics), and punctuation</li> <li>Recombine with NFKC normalization.</li> </ul> <p>Note</p> <p>See: Unicode normalization.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be collapsed.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Collapsed plain text.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def text_collapse(text):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize and simplify text for similarity hashing.\n    - Decompose with NFD normalization.\n    - Remove all whitespace characters and convert text to lower case\n    - Filter control characters, marks (diacritics), and punctuation\n    - Recombine with NFKC normalization.\n    !!! note\n        See: [Unicode normalization](https://unicode.org/reports/tr15/).\n    :param str text: Plain text to be collapsed.\n    :return: Collapsed plain text.\n    :rtype: str\n    \"\"\"\n# Decompose with NFD\ntext = unicodedata.normalize(\"NFD\", text)\n# Remove all whitespace and convert text to lower case\ntext = \"\".join(text.split()).lower()\n# Filter control characters, marks (diacritics), and punctuation\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] not in ic.core_opts.text_unicode_filter\n)\n# Recombine\ntext = unicodedata.normalize(\"NFKC\", text)\nreturn text\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.soft_hash_text_v0","title":"<code>soft_hash_text_v0(text)</code>","text":"<p>Creates a 256-bit similarity preserving hash for text input with algorithm v0.</p> <ul> <li>Slide over text with a   <code>text_ngram_size</code> wide window   and create <code>xxh32</code> hashes</li> <li>Create a <code>minhash_256</code> from the hashes generated   in the previous step.</li> </ul> <p>Note</p> <p>Before passing text to this function it must be:</p> <ul> <li>stripped of markup</li> <li>normalized</li> <li>stripped of whitespace</li> <li>lowercased</li> </ul> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be hashed.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit similarity preserving byte hash.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def soft_hash_text_v0(text):\n# type: (str) -&gt; bytes\n\"\"\"\n    Creates a 256-bit similarity preserving hash for text input with algorithm v0.\n    - Slide over text with a\n      [`text_ngram_size`][iscc_core.options.CoreOptions.text_ngram_size] wide window\n      and create [`xxh32`](https://cyan4973.github.io/xxHash/) hashes\n    - Create a [`minhash_256`][iscc_core.minhash.alg_minhash_256] from the hashes generated\n      in the previous step.\n    !!! note\n        Before passing text to this function it must be:\n        - stripped of markup\n        - normalized\n        - stripped of whitespace\n        - lowercased\n    :param str text: Plain text to be hashed.\n    :return: 256-bit similarity preserving byte hash.\n    :rtype: bytes\n    \"\"\"\nngrams = ic.sliding_window(text, ic.core_opts.text_ngram_size)\nfeatures = [xxhash.xxh32_intdigest(s.encode(\"utf-8\")) for s in ngrams]\nhash_digest = ic.alg_minhash_256(features)\nreturn hash_digest\n</code></pre>"},{"location":"units/content/code_content_video/","title":"ISCC - Video-Code","text":"<p>A similarity preserving hash for video content</p> <p>The Content-Code Video is generated from MPEG-7 video frame signatures. The iscc-sdk uses ffmpeg to extract frame signatures with the following command line parameters:</p> <p><code>$ ffmpeg -i video.mpg -vf fps=fps=5,signature=format=xml:filename=sig.xml -f null -</code></p> <p>The relevant frame signatures can be parsed from the following elements in sig.xml:</p> <p><code>&lt;FrameSignature&gt;0  0  0  1  0  0  1  0  1  1  0  0  1  1 ...&lt;/FrameSignature&gt;</code></p> <p>Tip</p> <p>It is also possible to extract the signatures in a more compact binary format. But the format requires a custom binary parser to decode the frame signaturs.</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code","title":"<code>gen_video_code(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code_v0","title":"<code>gen_video_code_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.soft_hash_video_v0","title":"<code>soft_hash_video_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Compute video hash v0 from MP7 frame signatures.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>2D matrix of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code>"},{"location":"utilities/utils/","title":"ISCC - Utilities","text":""},{"location":"utilities/utils/#iscc_core.utils.json_canonical","title":"<code>json_canonical(obj)</code>","text":"<p>Canonical, deterministic serialization of ISCC metadata.</p> <p>We serialize ISCC metadata in a deterministic/reproducible manner by using JCS (RFC 8785) canonicalization.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def json_canonical(obj):\n# type: (Any) -&gt; bytes\n\"\"\"\n    Canonical, deterministic serialization of ISCC metadata.\n    We serialize ISCC metadata in a deterministic/reproducible manner by using\n    [JCS (RFC 8785)](https://datatracker.ietf.org/doc/html/rfc8785) canonicalization.\n    \"\"\"\nser = jcs.canonicalize(obj)\ndes = json.loads(ser)\nif des != obj:\nraise ValueError(f\"Not canonicalizable {obj} round-trips to {des}\")\nreturn ser\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.sliding_window","title":"<code>sliding_window(seq, width)</code>","text":"<p>Generate a sequence of equal \"width\" slices each advancing by one elemnt.</p> <p>All types that have a length and can be sliced are supported (list, tuple, str ...). The result type matches the type of the input sequence. Fragment slices smaller than the width at the end of the sequence are not produced. If \"witdh\" is smaller than the input sequence than one element will be returned that is shorter than the requested width.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>Sequence</code> <p>Sequence of values to slide over</p> required <code>width</code> <code>int</code> <p>Width of sliding window in number of items</p> required <p>Returns:</p> Type Description <code>Generator</code> <p>A generator of window sized items</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def sliding_window(seq, width):\n# type: (Sequence, int) -&gt; Generator\n\"\"\"\n    Generate a sequence of equal \"width\" slices each advancing by one elemnt.\n    All types that have a length and can be sliced are supported (list, tuple, str ...).\n    The result type matches the type of the input sequence.\n    Fragment slices smaller than the width at the end of the sequence are not produced.\n    If \"witdh\" is smaller than the input sequence than one element will be returned that\n    is shorter than the requested width.\n    :param Sequence seq: Sequence of values to slide over\n    :param int width: Width of sliding window in number of items\n    :returns: A generator of window sized items\n    :rtype: Generator\n    \"\"\"\nif width &lt; 2:\nraise AssertionError(\"Sliding window width must be 2 or bigger.\")\nidx = range(max(len(seq) - width + 1, 1))\nreturn (seq[i : i + width] for i in idx)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_compare","title":"<code>iscc_compare(a, b)</code>","text":"<p>Calculate separate hamming distances of compatible components of two ISCCs</p> <p>Returns:</p> Type Description <code>dict</code> <p>A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_compare(a, b):\n# type: (str, str) -&gt; dict\n\"\"\"\n    Calculate separate hamming distances of compatible components of two ISCCs\n    :return: A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match\n    :rtype: dict\n    \"\"\"\nac = [ic.Code(unit) for unit in ic.iscc_decompose(a)]\nbc = [ic.Code(unit) for unit in ic.iscc_decompose(b)]\nresult = {}\nfor ca in ac:\nfor cb in bc:\ncat = (ca.maintype, ca.subtype, ca.version)\ncbt = (cb.maintype, cb.subtype, ca.version)\nif cat == cbt:\nif ca.maintype != ic.MT.INSTANCE:\nresult[ca.maintype.name.lower() + \"_dist\"] = iscc_distance_bytes(\nca.hash_bytes, cb.hash_bytes\n)\nelse:\nresult[\"instance_match\"] = ca.hash_bytes == cb.hash_bytes\nreturn result\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_similarity","title":"<code>iscc_similarity(a, b)</code>","text":"<p>Calculate similarity of ISCC codes as a percentage value (0-100).</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Similarity of ISCC a and b in percent (based on hamming distance)</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_similarity(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate similarity of ISCC codes as a percentage value (0-100).\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Similarity of ISCC a and b in percent (based on hamming distance)\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nhdist = iscc_distance_bytes(a, b)\nnbits = len(a) * 8\nsim = int(((nbits - hdist) / nbits) * 100)\nreturn sim\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance","title":"<code>iscc_distance(a, b)</code>","text":"<p>Calculate hamming distance of ISCC codes.</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distanced in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate hamming distance of ISCC codes.\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Hamming distanced in number of bits.\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nreturn iscc_distance_bytes(a, b)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance_bytes","title":"<code>iscc_distance_bytes(a, b)</code>","text":"<p>Calculate hamming distance for binary hash digests of equal length.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <code>bytes</code> <p>binary hash digest</p> required <code>b</code> <code>bytes</code> <p>binary hash digest</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distance in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance_bytes(a, b):\n# type: (bytes, bytes) -&gt; int\n\"\"\"\n    Calculate hamming distance for binary hash digests of equal length.\n    :param bytes a: binary hash digest\n    :param bytes b: binary hash digest\n    :return: Hamming distance in number of bits.\n    :rtype: int\n    \"\"\"\nif len(a) != len(b):\nraise AssertionError(f\"Hash diggest of unequal length: {len(a)} vs {len(b)}\")\nba, bb = bitarray(), bitarray()\nba.frombytes(a)\nbb.frombytes(b)\nreturn count_xor(ba, bb)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_pair_unpack","title":"<code>iscc_pair_unpack(a, b)</code>","text":"<p>Unpack two ISCC codes and return their body hash digests if their headers match.</p> <p>Headers match if their MainType, SubType, and Version are identical.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>Tuple[bytes, bytes]</code> <p>Tuple with hash digests of a and b</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If ISCC headers don\u00b4t match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_pair_unpack(a, b):\n# type: (str, str) -&gt; Tuple[bytes, bytes]\n\"\"\"\n    Unpack two ISCC codes and return their body hash digests if their headers match.\n    Headers match if their MainType, SubType, and Version are identical.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Tuple with hash digests of a and b\n    :rtype: Tuple[bytes, bytes]\n    :raise ValueError: If ISCC headers don\u00b4t match\n    \"\"\"\na, b = ic.iscc_clean(ic.iscc_normalize(a)), ic.iscc_clean(ic.iscc_normalize(b))\na, b = ic.decode_base32(a), ic.decode_base32(b)\na, b = ic.decode_header(a), ic.decode_header(b)\nif not a[:-1] == b[:-1]:\nraise ValueError(f\"ISCC headers don\u00b4t match: {a}, {b}\")\nreturn a[-1], b[-1]\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"ISCC - Codec &amp; Algorithms","text":"<p><code>iscc-core</code> is the reference implementation of the core algorithms of the ISCC (International Standard Content Code)</p>"},{"location":"#what-is-the-iscc","title":"What is the ISCC","text":"<p>The ISCC is a similarity preserving fingerprint and identifier for digital media assets.</p> <p>ISCCs are generated algorithmically from digital content, just like cryptographic hashes. However, instead of using a single cryptographic hash function to identify data only, the ISCC uses various algorithms to create a composite identifier that exhibits similarity-preserving properties (soft hash).</p> <p>The component-based structure of the ISCC identifies content at multiple levels of abstraction. Each component is self-describing, modular, and can be used separately or with others to aid in various content identification tasks. The algorithmic design supports content deduplication, database synchronization, indexing, integrity verification, timestamping, versioning, data provenance, similarity clustering, anomaly detection, usage tracking, allocation of royalties, fact-checking and general digital asset management use-cases.</p>"},{"location":"#what-is-iscc-core","title":"What is <code>iscc-core</code>","text":"<p><code>iscc-core</code> is a python based reference library of the core algorithms to create standard-compliant ISCC codes. It also a good reference for porting ISCC to other programming languages.</p> <p>Tip</p> <p>This is a low level reference implementation that does not inlcude features like mediatype detection, metadata extraction or file format specific content extraction. Please have a look at the iscc-sdk which adds those higher level features on top of the <code>iscc-core</code> library.</p>"},{"location":"#project-status","title":"Project Status","text":"<p>The ISCC is under development as ISO/CD 24138 - International Standard Content Code within ISO/TC 46/SC 9/WG 18.</p>"},{"location":"#iscc-architecture","title":"ISCC Architecture","text":""},{"location":"#iscc-maintypes","title":"ISCC MainTypes","text":"Idx Slug Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more components with common header"},{"location":"#installation","title":"Installation","text":"<p>Use the package manager pip to install <code>iscc-core</code>.</p> <pre><code>pip install iscc-core\n</code></pre>"},{"location":"#quick-start","title":"Quick Start","text":"<pre><code>import json\nimport iscc_core as ic\nmeta_code = ic.gen_meta_code(name=\"ISCC Test Document!\")\nprint(f\"Meta-Code:     {meta_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(meta_code['iscc'])}\\n\")\n# Extract text from file\nwith open(\"demo.txt\", \"rt\", encoding=\"utf-8\") as stream:\ntext = stream.read()\ntext_code = ic.gen_text_code_v0(text)\nprint(f\"Text-Code:     {text_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(text_code['iscc'])}\\n\")\n# Process raw bytes of textfile\nwith open(\"demo.txt\", \"rb\") as stream:\ndata_code = ic.gen_data_code(stream)\nprint(f\"Data-Code:     {data_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(data_code['iscc'])}\\n\")\nstream.seek(0)\ninstance_code = ic.gen_instance_code(stream)\nprint(f\"Instance-Code: {instance_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(instance_code['iscc'])}\\n\")\n# Combine ISCC-UNITs into ISCC-CODE\niscc_code = ic.gen_iscc_code(\n(meta_code[\"iscc\"], text_code[\"iscc\"], data_code[\"iscc\"], instance_code[\"iscc\"])\n)\n# Create convenience `Code` object from ISCC string\niscc_obj = ic.Code(iscc_code[\"iscc\"])\nprint(f\"ISCC-CODE:     {ic.iscc_normalize(iscc_obj.code)}\")\nprint(f\"Structure:     {iscc_obj.explain}\")\nprint(f\"Multiformat:   {iscc_obj.mf_base32}\\n\")\n# Compare with changed ISCC-CODE:\nnew_dc, new_ic = ic.Code.rnd(mt=ic.MT.DATA), ic.Code.rnd(mt=ic.MT.INSTANCE)\nnew_iscc = ic.gen_iscc_code((meta_code[\"iscc\"], text_code[\"iscc\"], new_dc.uri, new_ic.uri))\nprint(f\"Compare ISCC-CODES:\\n{iscc_obj.uri}\\n{new_iscc['iscc']}\")\nprint(json.dumps(ic.iscc_compare(iscc_obj.code, new_iscc[\"iscc\"]), indent=2))\n</code></pre> <p>The output of this example is as follows:</p> <pre><code>Meta-Code:     ISCC:AAAT4EBWK27737D2\nStructure:     META-NONE-V0-64-3e103656bffdfc7a\n\nText-Code:     ISCC:EAAQMBEYQF6457DP\nStructure:     CONTENT-TEXT-V0-64-060498817dcefc6f\n\nData-Code:     ISCC:GAA7UJMLDXHPPENG\nStructure:     DATA-NONE-V0-64-fa258b1dcef791a6\n\nInstance-Code: ISCC:IAA3Y7HR2FEZCU4N\nStructure:     INSTANCE-NONE-V0-64-bc7cf1d14991538d\n\nISCC-CODE:     ISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nStructure:     ISCC-TEXT-V0-MCDI-3e103656bffdfc7a060498817dcefc6ffa258b1dcef791a6bc7cf1d14991538d\nMultiformat:   bzqavabj6ca3fnp757r5ambeyqf6457dp7isywhoo66i2npd46hiutektru\n\nCompare ISCC-CODES:\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G7Y7HA2BMECKMVRBEQXR2BJOS6NA\n{\n  \"meta_dist\": 0,\n  \"content_dist\": 0,\n  \"data_dist\": 33,\n  \"instance_match\": false\n}\n</code></pre>"},{"location":"#documentation","title":"Documentation","text":"<p>Documentation is published athttps://core.iscc.codes</p>"},{"location":"#development","title":"Development","text":"<p>Requirements</p> <ul> <li>Python 3.7.2 or higher for code generation and static site building.</li> <li>Poetry for installation and dependency management.</li> </ul> <p>Development Setup</p> <pre><code>git clone https://github.com/iscc/iscc-core.git\ncd iscc-core\npoetry install\n</code></pre> <p>Development Tasks</p> <p>Tests, coverage, code formatting and other tasks can be run with the <code>poe</code> command:</p> <pre><code>poe\n\nPoe the Poet - A task runner that works well with poetry.\nversion 0.18.1\n\nResult: No task specified.\n\nUSAGE\n  poe [-h] [-v | -q] [--root PATH] [--ansi | --no-ansi] task [task arguments]\nGLOBAL OPTIONS\n  -h, --help     Show this help page and exit\n--version      Print the version and exit\n-v, --verbose  Increase command output (repeatable)\n-q, --quiet    Decrease command output (repeatable)\n-d, --dry-run  Print the task contents but don't actually run it\n  --root PATH    Specify where to find the pyproject.toml\n  --ansi         Force enable ANSI output\n  --no-ansi      Force disable ANSI output\nCONFIGURED TASKS\n  gentests       Generate conformance test data\n  format         Code style formating with black\n  docs           Copy README.md to /docs\n  format-md      Markdown formating with mdformat\n  lf             Convert line endings to lf\n  test           Run tests with coverage\n  sec            Security check with bandit\n  all\n</code></pre> <p>Use <code>poe all</code> to run all tasks before committing any changes.</p>"},{"location":"#maintainers","title":"Maintainers","text":"<p>@titusz</p>"},{"location":"#contributing","title":"Contributing","text":"<p>Pull requests are welcome. For significant changes, please open an issue first to discuss your plans. Please make sure to update tests as appropriate.</p> <p>You may also want join our developer chat on Telegram at https://t.me/iscc_dev.</p>"},{"location":"changelog/","title":"Changelog","text":""},{"location":"changelog/#106-2023-12-15","title":"[1.0.6] - 2023-12-15","text":"<ul> <li>Added wheel package testing to CI</li> <li>Fixed incompatible dependencies</li> </ul>"},{"location":"changelog/#105-2023-12-07","title":"[1.0.5] - 2023-12-07","text":"<ul> <li>Improved simhash performance</li> <li>Added native dct and wtahash support</li> <li>Added Python 3.12 support</li> <li>Updated and relax dependencies</li> </ul>"},{"location":"changelog/#104-2023-06-05","title":"[1.0.4] - 2023-06-05","text":"<ul> <li>Removed bases dependency</li> <li>Fixed mkdocstrings</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#103-2023-03-12","title":"[1.0.3] - 2023-03-12","text":"<ul> <li>Fix binary wheels</li> </ul>"},{"location":"changelog/#102-2023-03-12","title":"[1.0.2] - 2023-03-12","text":"<ul> <li>Publish binary wheels</li> </ul>"},{"location":"changelog/#101-2023-03-11","title":"[1.0.1] - 2023-03-11","text":"<ul> <li>Switch to standard bitarray module</li> <li>Switch tests to latest environments</li> <li>Add Python 3.11 support to TROVE classifiers</li> </ul>"},{"location":"changelog/#100-2023-01-24","title":"[1.0.0] - 2023-01-24","text":"<ul> <li>ISO/CD 24138 v1 Release</li> <li>Updateted dependencies</li> </ul>"},{"location":"changelog/#0214-2023-01-17","title":"[0.2.14] - 2023-01-17","text":"<ul> <li>Added ISO Reference documentation</li> <li>Removed non-standard conformance tests</li> <li>Fixed Meta-Code documentation</li> <li>Improved documentation CSS</li> </ul>"},{"location":"changelog/#0213-2023-01-16","title":"[0.2.13] - 2023-01-16","text":"<ul> <li>Added documentation for options</li> <li>Added python 3.11 support</li> <li>Added Markdown formating</li> <li>Added developmnet documentation</li> <li>Updated architecture figures</li> <li>Documentation cleanup</li> <li>Improved example code</li> <li>Optimized LF conversion</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0212-2022-11-24","title":"[0.2.12] - 2022-11-24","text":"<ul> <li>Fixed issue with data url compound media types</li> <li>Added ISCC version validation to <code>iscc_validate</code></li> <li>Added prefix check to <code>iscc_normalize</code></li> <li>Bundled fonts with documentation</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0211-2022-07-03","title":"[0.2.11] - 2022-07-03","text":"<ul> <li>Add support for gracefull build failures</li> </ul>"},{"location":"changelog/#0210-2022-07-03","title":"[0.2.10] - 2022-07-03","text":"<ul> <li>Fix pip instalation is missing setuptools</li> <li>Update mkdocs</li> </ul>"},{"location":"changelog/#029-2022-07-03","title":"[0.2.9] - 2022-07-03","text":"<ul> <li>Added iscc_compare function</li> <li>Optimized soft_hash_audio performance</li> <li>Removed Cython from build requirements</li> <li>Fixed api listing tool</li> <li>Updated codec architecture figure</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#028-2022-04-21","title":"[0.2.8] - 2022-04-21","text":"<ul> <li>Fixed bug with subtype for semantic code</li> <li>Changed URI representation to upper case</li> <li>Changed to disallow ISCC-ID creation from ISCC-IDs</li> <li>Added line conversion tool</li> <li>Removed source wheel distribution</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#027-2022-04-16","title":"[0.2.7] - 2022-04-16","text":"<ul> <li>Fixed bug in iscc_id_incr_v0</li> <li>Added support to accept ISCC-ID URI as input for iscc_id_incr_v0</li> <li>Added guard against custom subtype in random ISCC-CODE generation.</li> </ul>"},{"location":"changelog/#026-2022-04-13","title":"[0.2.6] - 2022-04-13","text":"<ul> <li>Added <code>KY</code> and <code>MM</code> to valid prefixes</li> <li>Added support to check for compiled extension modules</li> <li>Added universal wheel distribution</li> </ul>"},{"location":"changelog/#025-2022-04-10","title":"[0.2.5] - 2022-04-10","text":"<ul> <li>Fixed missing <code>jcs</code> dependency</li> <li>Added SubType <code>NONE</code> to MT.ISCC to distinquish from SUM</li> <li>Added support for deterministic generation of random ISCC-CODEs</li> <li>Added support for custom bit-sizes for random ISCC-CODEs</li> <li>Moved changelog into separate file</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#024-2022-03-19","title":"[0.2.4] - 2022-03-19","text":"<ul> <li>Updated dependencies</li> <li>Added Flake.from_int and Flake.from_string</li> <li>Made Flake comparable and hashable</li> <li>Use standard hex encoded multihash for datahash and metahash</li> </ul>"},{"location":"changelog/#023-2022-03-06","title":"[0.2.3] - 2022-03-06","text":"<ul> <li>Update to iscc-schema 0.3.3</li> <li>Change image normalization instructions</li> <li>Fix issue with exporting cdc cython only functions</li> </ul>"},{"location":"changelog/#021-2022-03-03","title":"[0.2.1] - 2022-03-03","text":"<ul> <li>Cleanup and update dependencies</li> <li>Fix bitarray api change</li> <li>Fix developer commands</li> </ul>"},{"location":"changelog/#020-2022-02-24","title":"[0.2.0] - 2022-02-24","text":"<ul> <li>Complete API refactoring</li> <li>Use Data-URL as input for Meta-Code</li> <li>Use wallet address for ISCC-ID creation</li> <li>Added new Flake-Code (distributed time/random ID)</li> <li>Replaced assertions with exeptions</li> <li>Use secure random functions</li> <li>Retired Python 3.6 support (EOL)</li> <li>Return simple <code>dict</code> objects from generator functions</li> <li>Added ISCC string validation</li> <li>Added multiple helper functions</li> </ul>"},{"location":"changelog/#019-2021-12-17","title":"[0.1.9] - 2021-12-17","text":"<ul> <li>Added warning on non-standard options</li> <li>Added multiformats support</li> <li>Added uri representation</li> <li>Removed redundant cdc_avg_chunk_size option</li> <li>Updated codec format documentation</li> </ul>"},{"location":"changelog/#018-2021-12-12","title":"[0.1.8] - 2021-12-12","text":"<ul> <li>Added conformance tests for all top level functions</li> <li>Added conformance tests to source dir</li> <li>Added conformance module with <code>selftest</code> function</li> <li>Changed gen_image_code to accept normalized pixels instead of stream</li> <li>Changed opts to core_opts</li> <li>Removed image pre-processing and Pillow dependency</li> <li>Fixed readability of conformance tests</li> <li>Fixed soft_hash_video_v0 to accept non-tuple sequences</li> <li>Updated example code</li> </ul>"},{"location":"changelog/#017-2021-12-09","title":"[0.1.7] - 2021-12-09","text":"<ul> <li>Add dotenv for enviroment based configuration</li> <li>Cleanup package toplevel imports</li> <li>Return schema objects for iscc_code and iscc_id</li> <li>Exclude unset and none values from result dicts</li> <li>Add support for multiple code combinations for ISCC-CODE</li> <li>Add support for ISCC-ID based on singular Instance-Code</li> <li>Add initial conformance test system</li> </ul>"},{"location":"changelog/#016-2021-11-29","title":"[0.1.6] - 2021-11-29","text":"<ul> <li>Show counter for ISCC-ID in Code.explain</li> </ul>"},{"location":"changelog/#015-2021-11-28","title":"[0.1.5] - 2021-11-28","text":"<ul> <li>Fix documentation</li> <li>Change metahash creation logic</li> <li>Refactor models</li> <li>Add Content-Code-Mixed</li> <li>Add ISCC-ID</li> <li>Refactor <code>compose</code> to <code>gen_iscc_code</code></li> <li>Refactor <code>models</code> to <code>schema</code></li> </ul>"},{"location":"changelog/#014-2021-11-17","title":"[0.1.4] - 2021-11-17","text":"<ul> <li>Simplified options</li> <li>Optimize video WTA-hash for use with 64-bit granular features</li> </ul>"},{"location":"changelog/#013-2021-11-15","title":"[0.1.3] - 2021-11-15","text":"<ul> <li>Try to compile Cython/C accelerator modules when installing via pip</li> <li>Simplify soft_hash api return values</li> <li>Add .code() method to InstanceHasher, DataHasher</li> <li>Remove granular fingerprint calculation</li> <li>Add more top-level imports</li> </ul>"},{"location":"changelog/#012-2021-11-14","title":"[0.1.2] - 2021-11-14","text":"<ul> <li>Export more functions to toplevel</li> <li>Return schema driven objects from ISCC code generators.</li> </ul>"},{"location":"changelog/#011-2021-11-14","title":"[0.1.1] - 2021-11-14","text":"<ul> <li>Fix packaging problems</li> </ul>"},{"location":"changelog/#010-2021-11-13","title":"[0.1.0] - 2021-11-13","text":"<ul> <li>Initial release</li> </ul>"},{"location":"conformance/","title":"ISCC - Conformance Testing","text":"<p>An application that claims ISCC conformance MUST pass all core functions from the ISCC conformance test suite. The test suite is available as JSON data on GitHub. Test data is structured as follows:</p> <pre><code>{\n\"&lt;function_name&gt;\": {\n\"&lt;test_name&gt;\": {\n\"inputs\": [\"&lt;value1&gt;\", \"&lt;value2&gt;\"],\n\"outputs\": [\"value1&gt;\", \"&lt;value2&gt;\"]\n}\n}\n}\n</code></pre> <p>Inputs that are expected to be <code>raw bytes or byte-streams</code> are embedded as HEX encoded strings in JSON and prefixed with <code>stream:</code> or <code>bytes</code> to support automated decoding during implementation testing.</p> <p>Example</p> <p>Byte-stream outputs in JSON test data: <pre><code>\"gen_data_code_v0\": {\n\"test_0000_two_bytes_64\": {\n\"inputs\": [\n\"stream:ff00\",\n64\n],\n\"outputs\": {\n\"iscc\": \"GAAXL2XYM5BQIAZ3\"\n}\n},\n...\n</code></pre></p>"},{"location":"conformance/#iscc_core.conformance.conformance_testdata","title":"<code>conformance_testdata()</code>","text":"<p>Yield tuples of test data.</p> <p>Returns:</p> Type Description <code>Generator[Tuple[str, Callable, List[Any], List[Any]]]</code> <p>Tuple with testdata (test_name, func_obj, inputs, outputs)</p>"},{"location":"conformance/#iscc_core.conformance.conformance_selftest","title":"<code>conformance_selftest()</code>","text":"<p>Run conformance tests.</p> <p>Returns:</p> Type Description <code>bool</code> <p>whether all tests passed</p>"},{"location":"constants/","title":"ISCC - Types and Constants","text":""},{"location":"constants/#iscc_core.constants.MT","title":"<code>MT</code>","text":""},{"location":"constants/#iscc_core.constants.MT--mt-maintypes","title":"MT - MainTypes","text":"Uint Symbol Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more ISCC-UNITs with common header"},{"location":"constants/#iscc_core.constants.ST","title":"<code>ST</code>","text":""},{"location":"constants/#iscc_core.constants.ST--st-subtypes","title":"ST - SubTypes","text":"Uint Symbol Bits Purpose 0 NONE 0000 For MainTypes that do not specify SubTypes"},{"location":"constants/#iscc_core.constants.ST_CC","title":"<code>ST_CC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_CC--st_cc","title":"ST_CC","text":"<p>SubTypes for <code>MT.CONTENT</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Match on syntactic text similarity 1 IMAGE 0001 Match on perceptual image similarity 2 AUDIO 0010 Match on audio chroma similarity 3 VIDEO 0011 Match on perceptual similarity 4 MIXED 0100 Match on similarity of content codes"},{"location":"constants/#iscc_core.constants.ST_ISCC","title":"<code>ST_ISCC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_ISCC--st_iscc","title":"ST_ISCC","text":"<p>SubTypes for <code>MT.ISCC</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Composite ISCC inlcuding Text-Code 1 IMAGE 0001 Composite ISCC inlcuding Image-Code 2 AUDIO 0010 Composite ISCC inlcuding Audio-Code 3 VIDEO 0011 Composite ISCC inlcuding Video-Code 4 MIXED 0100 Composite ISCC inlcuding Mixed-Code 5 SUM 0101 Composite ISCC inlcuding only Data- and Instance-Code 6 NONE 0110 Composite ISCC including Meta, Data and Instance-Code"},{"location":"constants/#iscc_core.constants.VS","title":"<code>VS</code>","text":""},{"location":"constants/#iscc_core.constants.VS--vs-version","title":"VS - Version","text":"<p>Code Version</p> Uint Symbol Bits Purpose 0 V0 0000 Initial Version of Code without breaking changes"},{"location":"constants/#iscc_core.constants.LN","title":"<code>LN</code>","text":""},{"location":"constants/#iscc_core.constants.LN--ln-length","title":"LN - Length","text":"<p>Valid lengths for hash-digests.</p> <ul> <li>L32 = 32</li> <li>L64 = 64</li> <li>L72 = 72</li> <li>L80 = 80</li> <li>L96 = 96</li> <li>L128 = 128</li> <li>L160 = 160</li> <li>L192 = 192</li> <li>L224 = 224</li> <li>L256 = 256</li> </ul>"},{"location":"constants/#iscc_core.constants.MULTIBASE","title":"<code>MULTIBASE</code>","text":"<p>Supported Multibase encodings.</p> <ul> <li>base16 -&gt; f</li> <li>base32 -&gt; b</li> <li>base32hex -&gt; v</li> <li>base58btc -&gt; z</li> <li>base64url -&gt; u</li> </ul>"},{"location":"iscc_code/","title":"ISCC-CODE","text":"<p>A multi-component identifier for digital media assets.</p> <p>An ISCC-CODE can be generated from the concatenation of the digests of the following five ISCC-UNITs together with a single common header:</p> <ul> <li>Meta-Code - Encodes metadata similarity</li> <li>Semantic-Code - Encodes semantic content similarity (to be developed)</li> <li>Content-Code - Encodes syntactic/perceptual similarity</li> <li>Data-Code - Encodes raw bitstream similarity</li> <li>Instance-Code - Data checksum</li> </ul> <p>The following sequences of ISCC-UNITs are possible:</p> <ul> <li>Data, Instance</li> <li>Content, Data, Instance</li> <li>Semantic, Data, Instance</li> <li>Content, Semantic, Data, Instance</li> <li>Meta, Data, Instance</li> <li>Meta, Content, Data, Instance</li> <li>Meta, Semantic, Data, Instance</li> <li>Meta, Semantic, Content, Data, Instance</li> </ul>"},{"location":"iscc_code/#iscc_core.iscc_code.gen_iscc_code_v0","title":"<code>gen_iscc_code_v0(codes)</code>","text":"<p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iscc_id/","title":"ISCC-ID","text":"<p>A decentralized, owned, and short identifier for digital assets.</p> <p>The ISCC-ID is generated from a similarity-hash of the units of an ISCC-CODE together with a blockchain wallet address. Its SubType designates the blockchain from which the ISCC-ID was minted. The similarity-hash is always at least 64-bits and optionally suffixed with a <code>uvarint</code> endcoded <code>uniqueness counter</code>. The <code>uniqueness counter</code> is added and incremented only if the mint colides with a pre-existing ISCC-ID minted from the same blockchain from a different ISCC-CODE or from an identical ISCC-CODE registered by a different signatory.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id","title":"<code>gen_iscc_id(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate  ISCC-ID from ISCC-CODE with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id_v0","title":"<code>gen_iscc_id_v0(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate an ISCC-ID from an ISCC-CODE with uniqueness counter 'uc' with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.soft_hash_iscc_id_v0","title":"<code>soft_hash_iscc_id_v0(iscc_code, wallet, uc = 0)</code>","text":"<p>Calculate ISCC-ID hash digest from ISCC-CODE with algorithm v0.</p> <p>Accepts an ISCC-CODE or any sequence of ISCC-UNITs.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>ISCC-CODE</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter for ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Digest for ISCC-ID without header but including uniqueness counter.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr","title":"<code>iscc_id_incr(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr_v0","title":"<code>iscc_id_incr_v0(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one (without \"ISCC:\" prefix).</p>"},{"location":"iscc_id/#iscc_core.iscc_id.alg_simhash_from_iscc_id","title":"<code>alg_simhash_from_iscc_id(iscc_id, wallet)</code>","text":"<p>Extract similarity preserving hex-encoded hash digest from ISCC-ID</p> <p>We need to un-xor the ISCC-ID hash digest with the wallet address hash to obtain the similarity preserving bytestring.</p>"},{"location":"iso-reference/","title":"ISCC - ISO Reference","text":"<p>The following functions are the reference implementations of ISO 24138:</p>"},{"location":"iso-reference/#iso-24138-51-meta-code","title":"ISO 24138 / 5.1 Meta-Code","text":"<code>gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits)</code> # <p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"iso-reference/#iso-24138-53-text-code","title":"ISO 24138 / 5.3 Text-Code","text":"<code>gen_text_code_v0(text, bits=ic.core_opts.text_bits)</code> # <p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"iso-reference/#iso-24138-54-image-code","title":"ISO 24138 / 5.4 Image-Code","text":"<code>gen_image_code_v0(pixels, bits=ic.core_opts.image_bits)</code> # <p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p> Source code in <code>iscc_core\\code_content_image.py</code> <pre><code>def gen_image_code_v0(pixels, bits=ic.core_opts.image_bits):\n# type: (Sequence[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Image with algorithm v0.\n    :param Sequence[int] pixels: Normalized image pixels (32x32 flattened gray values)\n    :param int bits: Bit-length of ISCC Content-Code Image (default 64).\n    :return: ISCC object with Content-Code Image.\n    :rtype: ISCC\n    \"\"\"\ndigest = soft_hash_image_v0(pixels, bits=bits)\nimage_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.IMAGE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + image_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-55-audio-code","title":"ISO 24138 / 5.5 Audio-Code","text":"<code>gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits)</code> # <p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p> Source code in <code>iscc_core\\code_content_audio.py</code> <pre><code>def gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits):\n# type: (Iterable[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Audio with algorithm v0.\n    :param Iterable[int] cv: Chromaprint vector\n    :param int bits: Bit-length resulting Content-Code Audio (multiple of 64)\n    :return: ISCC object with Content-Code Audio\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_audio_v0(cv, bits=bits)\naudio_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.AUDIO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + audio_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-56-video-code","title":"ISO 24138 / 5.6 Video-Code","text":"<code>gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits)</code> # <p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p> Source code in <code>iscc_core\\code_content_video.py</code> <pre><code>def gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits):\n# type: (Sequence[ic.FrameSig], int) -&gt; dict\n\"\"\"\n    Create an ISCC Video-Code with algorithm v0.\n    :param ic.FrameSig frame_sigs: Sequence of MP7 frame signatures\n    :param int bits: Bit-length resulting Video-Code (multiple of 64)\n    :return: ISCC object with Video-Code\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_video_v0(frame_sigs, bits=bits)\nvideo_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.VIDEO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + video_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-57-mixed-code","title":"ISO 24138 / 5.7 Mixed-Code","text":"<code>gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits)</code> # <p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p> Source code in <code>iscc_core\\code_content_mixed.py</code> <pre><code>def gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits):\n# type: (Sequence[str], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code-Mixed with algorithm v0.\n    If the provided codes are of mixed length they are stripped to `bits` length for\n    calculation.\n    :param Iterable[str] codes: a list of Content-Codes.\n    :param int bits: Target bit-length of generated Content-Code-Mixed.\n    :return: ISCC object with Content-Code Mixed.\n    :rtype: dict\n    \"\"\"\ndigests = [ic.decode_base32(ic.iscc_clean(code)) for code in codes]\ndigest = soft_hash_codes_v0(digests, bits=bits)\nmixed_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.MIXED,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + mixed_code\nreturn dict(iscc=iscc, parts=list(codes))\n</code></pre>"},{"location":"iso-reference/#iso-24138-58-data-code","title":"ISO 24138 / 5.8 Data-Code","text":"<code>gen_data_code_v0(stream, bits=ic.core_opts.data_bits)</code> # <p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p> Source code in <code>iscc_core\\code_data.py</code> <pre><code>def gen_data_code_v0(stream, bits=ic.core_opts.data_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Data-Code with algorithm v0.\n    :param Stream stream: Input data stream.\n    :param int bits: Bit-length of ISCC Data-Code (default 64).\n    :return: ISCC object with Data-Code\n    :rtype: dict\n    \"\"\"\nhasher = DataHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ndata_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + data_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-59-instance-code","title":"ISO 24138 / 5.9 Instance-Code","text":"<code>gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits)</code> # <p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p> Source code in <code>iscc_core\\code_instance.py</code> <pre><code>def gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Instance-Code with algorithm v0.\n    :param Stream stream: Binary data stream for Instance-Code generation\n    :param int bits: Bit-length of resulting Instance-Code (multiple of 64)\n    :return: ISCC object with Instance-Code and properties: datahash, filesize\n    :rtype: dict\n    \"\"\"\nhasher = InstanceHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ninstance_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + instance_code\ninstance_code_obj = dict(\niscc=iscc,\ndatahash=hasher.multihash(),\nfilesize=hasher.filesize,\n)\nreturn instance_code_obj\n</code></pre>"},{"location":"iso-reference/#iso-24138-60-iscc-code","title":"ISO 24138 / 6.0 ISCC-CODE","text":"<code>gen_iscc_code_v0(codes)</code> # <p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"algorithms/cdc/","title":"ISCC - Content Defined Chunking","text":"<p>Compatible with fastcdc</p>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_chunks","title":"<code>alg_cdc_chunks(data, utf32, avg_chunk_size = ic.core_opts.data_avg_chunk_size)</code>","text":"<p>A generator that yields data-dependent chunks for <code>data</code>.</p> <p>Usage Example:</p> <pre><code>for chunk in cdc_data_chunks(data):\nhash(chunk)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>Raw data for variable sized chunking.</p> required <code>utf32</code> <code>bool</code> <p>If true assume we are chunking text that is utf32 encoded.</p> required <code>avg_chunk_size</code> <code>int</code> <p>Target chunk size in number of bytes.</p> <code>ic.core_opts.data_avg_chunk_size</code> <p>Returns:</p> Type Description <code>Generator[bytes]</code> <p>A generator that yields data chunks of variable sizes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_chunks(data, utf32, avg_chunk_size=ic.core_opts.data_avg_chunk_size):\n# type: (Data, bool, int) -&gt; Generator[bytes, None, None]\n\"\"\"\n    A generator that yields data-dependent chunks for `data`.\n    Usage Example:\n    ```python\n    for chunk in cdc_data_chunks(data):\n        hash(chunk)\n    ```\n    :param bytes data: Raw data for variable sized chunking.\n    :param bool utf32: If true assume we are chunking text that is utf32 encoded.\n    :param int avg_chunk_size: Target chunk size in number of bytes.\n    :return: A generator that yields data chunks of variable sizes.\n    :rtype: Generator[bytes]\n    \"\"\"\nstream = io.BytesIO(data)\nbuffer = stream.read(ic.core_opts.io_read_size)\nif not buffer:\nyield b\"\"\nmi, ma, cs, mask_s, mask_l = alg_cdc_params(avg_chunk_size)\nbuffer = memoryview(buffer)\nwhile buffer:\nif len(buffer) &lt;= ma:\nbuffer = memoryview(bytes(buffer) + stream.read(ic.core_opts.io_read_size))\ncut_point = alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)\n# Make sure cut points are at 4-byte aligned for utf32 encoded text\nif utf32:\ncut_point -= cut_point % 4\nyield bytes(buffer[:cut_point])\nbuffer = buffer[cut_point:]\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_offset","title":"<code>alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)</code>","text":"<p>Find breakpoint offset for a given buffer.</p> <p>Parameters:</p> Name Type Description Default <code>buffer</code> <code>Data</code> <p>The data to be chunked.</p> required <code>mi</code> <code>int</code> <p>Minimum chunk size.</p> required <code>ma</code> <code>int</code> <p>Maximung chunk size.</p> required <code>cs</code> <code>int</code> <p>Center size.</p> required <code>mask_s</code> <code>int</code> <p>Small mask.</p> required <code>mask_l</code> <code>int</code> <p>Large mask.</p> required <p>Returns:</p> Type Description <code>int</code> <p>Offset of dynamic cutpoint in number of bytes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l):\n# type: (ic.Data, int, int, int, int, int) -&gt; int\n\"\"\"\n    Find breakpoint offset for a given buffer.\n    :param Data buffer: The data to be chunked.\n    :param int mi: Minimum chunk size.\n    :param int ma: Maximung chunk size.\n    :param int cs: Center size.\n    :param int mask_s: Small mask.\n    :param int mask_l: Large mask.\n    :return: Offset of dynamic cutpoint in number of bytes.\n    :rtype: int\n    \"\"\"\npattern = 0\ni = mi\nsize = len(buffer)\nbarrier = min(cs, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_s:\nreturn i + 1\ni += 1\nbarrier = min(ma, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_l:\nreturn i + 1\ni += 1\nreturn i\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_params","title":"<code>alg_cdc_params(avg_size: int) -&gt; tuple</code>","text":"<p>Calculate CDC parameters</p> <p>Parameters:</p> Name Type Description Default <code>avg_size</code> <code>int</code> <p>Target average size of chunks in number of bytes.</p> required <p>Returns:</p> Type Description <code>tuple</code> <p>Tuple of (min_size, max_size, center_size, mask_s, mask_l).</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_params(avg_size: int) -&gt; tuple:\n\"\"\"\n    Calculate CDC parameters\n    :param int avg_size: Target average size of chunks in number of bytes.\n    :returns: Tuple of (min_size, max_size, center_size, mask_s, mask_l).\n    \"\"\"\nceil_div = lambda x, y: (x + y - 1) // y\nmask = lambda b: 2**b - 1\nmin_size = avg_size // 4\nmax_size = avg_size * 8\noffset = min_size + ceil_div(min_size, 2)\ncenter_size = avg_size - offset\nbits = round(log2(avg_size))\nmask_s = mask(bits + 1)\nmask_l = mask(bits - 1)\nreturn min_size, max_size, center_size, mask_s, mask_l\n</code></pre>"},{"location":"algorithms/dct/","title":"ISCC - Discrete Cosine Transform","text":""},{"location":"algorithms/dct/#iscc_core.dct.alg_dct","title":"<code>alg_dct(v)</code>","text":"<p>Discrete cosine transform.</p> <p>See: nayuki.io.</p> <p>Parameters:</p> Name Type Description Default <code>v</code> <code>Sequence[float]</code> <p>Input vector for DCT calculation.</p> required <p>Returns:</p> Type Description <code>List</code> <p>DCT Transformed vector.</p> Source code in <code>iscc_core\\dct.py</code> <pre><code>def alg_dct(v):\n# type: (Sequence[float]) -&gt; List\n\"\"\"\n    Discrete cosine transform.\n    See: [nayuki.io](https://www.nayuki.io/page/fast-discrete-cosine-transform-algorithms).\n    :param Sequence[float] v: Input vector for DCT calculation.\n    :return: DCT Transformed vector.\n    :rtype: List\n    \"\"\"\nn = len(v)\nif n == 1:\nreturn list(v)\nelif n == 0 or n % 2 != 0:\nraise ValueError()\nelse:\nhalf = n // 2\nalpha = [(v[i] + v[-(i + 1)]) for i in range(half)]\nbeta = [\n(v[i] - v[-(i + 1)]) / (math.cos((i + 0.5) * math.pi / n) * 2.0) for i in range(half)\n]\nalpha = alg_dct(alpha)\nbeta = alg_dct(beta)\nresult = []\nfor i in range(half - 1):\nresult.append(alpha[i])\nresult.append(beta[i] + beta[i + 1])\nresult.append(alpha[-1])\nresult.append(beta[-1])\nreturn result\n</code></pre>"},{"location":"algorithms/minhash/","title":"ISCC - Minhash","text":""},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash","title":"<code>alg_minhash(features)</code>","text":"<p>Calculate a 64 dimensional minhash integer vector.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>List[int]</code> <p>Minhash vector</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash(features):\n# type: (List[int]) -&gt; List[int]\n\"\"\"\n    Calculate a 64 dimensional minhash integer vector.\n    :param List[int] features: List of integer features\n    :return: Minhash vector\n    :rtype: List[int]\n    \"\"\"\nreturn [\nmin([(((a * f + b) &amp; MAXI64) % MPRIME) &amp; MAXH for f in features]) for a, b in zip(MPA, MPB)\n]\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_64","title":"<code>alg_minhash_64(features)</code>","text":"<p>Create 64-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>64-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_64(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 64-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 64-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 1)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_256","title":"<code>alg_minhash_256(features)</code>","text":"<p>Create 256-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_256(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 256-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 4)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_compress","title":"<code>alg_minhash_compress(mhash, lsb = 4)</code>","text":"<p>Compress minhash vector to byte hash-digest.</p> <p>Concatenates <code>lsb</code> number of least-significant bits from each integer in <code>mhash</code>. For example an <code>mhash</code> with 64 integers and <code>lsb=4</code> will produce a 256-bit summary of the minhash vector.</p> <p>Parameters:</p> Name Type Description Default <code>mhash</code> <code>List[int]</code> <p>List of minhash integer features</p> required <code>lsb</code> <code>int</code> <p>Number of the least significant bits to retain</p> <code>4</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_compress(mhash, lsb=4):\n# type: (List[int], int) -&gt; bytes\n\"\"\"\n    Compress minhash vector to byte hash-digest.\n    Concatenates `lsb` number of least-significant bits from each integer in `mhash`.\n    For example an `mhash` with 64 integers and `lsb=4` will produce a 256-bit summary\n    of the minhash vector.\n    :param List[int] mhash: List of minhash integer features\n    :param int lsb: Number of the least significant bits to retain\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nbits: str = \"\"\nfor bitpos in range(lsb):\nfor h in mhash:\nbits += str(h &gt;&gt; bitpos &amp; 1)\nreturn int(bits, 2).to_bytes((len(bits) + 7) // 8, \"big\")\n</code></pre>"},{"location":"algorithms/simhash/","title":"ISCC - Simhash","text":""},{"location":"algorithms/simhash/#iscc_core.simhash.alg_simhash","title":"<code>alg_simhash(hash_digests)</code>","text":"<p>Creates a similarity preserving hash from a sequence of equal sized hash digests.</p> <p>Parameters:</p> Name Type Description Default <code>hash_digests</code> <code>list</code> <p>A sequence of equaly sized byte-hashes.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity byte-hash</p> Source code in <code>iscc_core\\simhash.py</code> <pre><code>def alg_simhash(hash_digests):\n# type: (list[bytes]) -&gt; bytes\n\"\"\"\n    Creates a similarity preserving hash from a sequence of equal sized hash digests.\n    :param list hash_digests: A sequence of equaly sized byte-hashes.\n    :returns: Similarity byte-hash\n    :rtype: bytes\n    \"\"\"\nn_bytes = len(hash_digests[0])\nn_bits = n_bytes * 8\nvector = [0] * n_bits\nfor digest in hash_digests:\nh = bitarray()\nh.frombytes(digest)\nfor i in range(n_bits):\nvector[i] += h[i]\nminfeatures = len(hash_digests) / 2\nshash = 0\nfor i in range(n_bits):\nif vector[i] &gt;= minfeatures:\nshash |= 1 &lt;&lt; (n_bits - 1 - i)\nreturn shash.to_bytes(n_bytes, \"big\")\n</code></pre>"},{"location":"algorithms/wtahash/","title":"ISCC - Winner Takes All Hash","text":""},{"location":"algorithms/wtahash/#iscc_core.wtahash.alg_wtahash","title":"<code>alg_wtahash(vec: Sequence[float], bits: Sequence[float]) -&gt; bytes</code>","text":"<p>Calculate WTA Hash for vector with 380 values (MP7 frame signature).</p> Source code in <code>iscc_core\\wtahash.py</code> <pre><code>def alg_wtahash(vec: Sequence[float], bits) -&gt; bytes:\n\"\"\"Calculate WTA Hash for vector with 380 values (MP7 frame signature).\"\"\"\nh = []\nfor perm in WTA_VIDEO_ID_PERMUTATIONS:\nv = vec[perm[0]], vec[perm[1]]\nh.append(v.index(max(v)))\nif len(h) == bits:\nbreak\nh = bitarray(h).tobytes()\nreturn h\n</code></pre>"},{"location":"codec/","title":"ISCC - Codec","text":"<p>This module implements encoding, decoding and transcoding functions of ISCC</p>"},{"location":"codec/#codec-overview","title":"Codec Overview","text":""},{"location":"codec/#codec-functions","title":"Codec Functions","text":""},{"location":"codec/#iscc_core.codec.encode_component","title":"<code>encode_component(mtype, stype, version, bit_length, digest)</code>","text":"<p>Encode an ISCC unit inlcuding header and body with standard base32 encoding.</p> <p>Note</p> <p>The <code>length</code> value must be the length in number of bits for the component. If <code>digest</code> has more bits than specified by <code>length</code> it wil be truncated.</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>Maintype of unit (0-6)</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit depending on MainType (0-5)</p> required <code>version</code> <code>Version</code> <p>Version of unit algorithm (0).</p> required <code>bit_length</code> <code>length</code> <p>Length of unit, in number of bits (multiple of 32)</p> required <code>digest</code> <code>bytes</code> <p>The hash digest of the unit.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32 encoded ISCC-UNIT.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_component(mtype, stype, version, bit_length, digest):\n# type: (MainType, SubType, Version, Length, bytes) -&gt; str\n\"\"\"\n    Encode an ISCC unit inlcuding header and body with standard base32 encoding.\n    !!! note\n        The `length` value must be the **length in number of bits** for the component.\n        If `digest` has more bits than specified by `length` it wil be truncated.\n    :param MainType mtype: Maintype of unit (0-6)\n    :param SubType stype: SubType of unit depending on MainType (0-5)\n    :param Version version: Version of unit algorithm (0).\n    :param length bit_length: Length of unit, in number of bits (multiple of 32)\n    :param bytes digest: The hash digest of the unit.\n    :return: Base32 encoded ISCC-UNIT.\n    :rtype: str\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.ID, MT.FLAKE):\nencoded_length = encode_length(mtype, bit_length)\nelif mtype == MT.ISCC:\nraise ValueError(f\"{mtype} is not a unit\")\nelse:\nraise ValueError(f\"Illegal MainType {mtype}\")\nnbytes = bit_length // 8\nheader = encode_header(mtype, stype, version, encoded_length)\nbody = digest[:nbytes]\ncomponent_code = encode_base32(header + body)\nreturn component_code\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_header","title":"<code>encode_header(mtype, stype, version = 0, length = 1)</code>","text":"<p>Encodes header values with nibble-sized (4-bit) variable-length encoding. The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles is uneven it is padded with 4-bit <code>0000</code> at the end.</p> <p>Warning</p> <p>The length value must be encoded beforhand because its semantics depend on the MainType (see <code>encode_length</code> function).</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>MainType of unit.</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit.</p> required <code>version</code> <code>Version</code> <p>Version of component algorithm.</p> <code>0</code> <code>length</code> <code>Length</code> <p>length value of unit (1 means 64-bits for standard units)</p> <code>1</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Varnibble stream encoded ISCC header as bytes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_header(mtype, stype, version=0, length=1):\n# type: (MainType, SubType, Version, Length) -&gt; bytes\n\"\"\"\n    Encodes header values with nibble-sized (4-bit) variable-length encoding.\n    The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles\n    is uneven it is padded with 4-bit `0000` at the end.\n    !!! warning\n        The length value must be encoded beforhand because its semantics depend on\n        the MainType (see `encode_length` function).\n    :param MainType mtype: MainType of unit.\n    :param SubType stype: SubType of unit.\n    :param Version version: Version of component algorithm.\n    :param Length length: length value of unit (1 means 64-bits for standard units)\n    :return: Varnibble stream encoded ISCC header as bytes.\n    :rtype: bytes\n    \"\"\"\n# TODO verify that all header params and there combination is valid\nheader = bitarray()\nfor n in (mtype, stype, version, length):\nheader += encode_varnibble(n)\n# Append zero-padding if required (right side, least significant bits).\nheader.fill()\nreturn header.tobytes()\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_header","title":"<code>decode_header(data)</code>","text":"<p>Decodes varnibble encoded header and returns it together with <code>tail data</code>.</p> <p>Tail data is included to enable decoding of sequential ISCCs. The returned tail data must be truncated to decode_length(r[0], r[3]) bits to recover the actual hash-bytes.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>ISCC bytes</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>(MainType, SubType, Version, length, TailData)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_header(data):\n# type: (bytes) -&gt; IsccTuple\n\"\"\"\n    Decodes varnibble encoded header and returns it together with `tail data`.\n    Tail data is included to enable decoding of sequential ISCCs. The returned tail\n    data must be truncated to decode_length(r[0], r[3]) bits to recover the actual\n    hash-bytes.\n    :param bytes data: ISCC bytes\n    :return: (MainType, SubType, Version, length, TailData)\n    :rtype: IsccTuple\n    \"\"\"\nresult = []\nba = bitarray()\nba.frombytes(data)\ndata = ba\nfor _ in range(4):\nvalue, data = decode_varnibble(data)\nresult.append(value)\n# Strip 4-bit padding if required\nif len(data) % 8 and data[:4] == bitarray(\"0000\"):\ndata = data[4:]\nresult.append(data.tobytes())\nreturn tuple(result)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_varnibble","title":"<code>encode_varnibble(n)</code>","text":"<p>Writes integer to variable length sequence of 4-bit chunks.</p> <p>Variable-length encoding scheme:</p> prefix bits nibbles data bits unsigned range 0 1 3 0 - 7 10 2 6 8 - 71 110 3 9 72 - 583 1110 4 12 584 - 4679 <p>Parameters:</p> Name Type Description Default <code>n</code> <code>int</code> <p>Positive integer to be encoded as varnibble (0-4679)</p> required <p>Returns:</p> Type Description <code>bitarray</code> <p>Varnibble encoded integera</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_varnibble(n):\n# type: (int) -&gt; bitarray\n\"\"\"\n    Writes integer to variable length sequence of 4-bit chunks.\n    Variable-length encoding scheme:\n    ------------------------------------------------------\n    | prefix bits | nibbles | data bits | unsigned range |\n    | ----------- | ------- | --------- | -------------- |\n    | 0           | 1       | 3         | 0 - 7          |\n    | 10          | 2       | 6         | 8 - 71         |\n    | 110         | 3       | 9         | 72 - 583       |\n    | 1110        | 4       | 12        | 584 - 4679     |\n    :param int n: Positive integer to be encoded as varnibble (0-4679)\n    :return: Varnibble encoded integera\n    :rtype: bitarray\n    \"\"\"\nif 0 &lt;= n &lt; 8:\nreturn int2ba(n, length=4)\nelif 8 &lt;= n &lt; 72:\nreturn bitarray(\"10\") + int2ba(n - 8, length=6)\nelif 72 &lt;= n &lt; 584:\nreturn bitarray(\"110\") + int2ba(n - 72, length=9)\nelif 584 &lt;= n &lt; 4680:\nreturn bitarray(\"1110\") + int2ba(n - 584, length=12)\nelse:\nraise ValueError(\"Value must be between 0 and 4679\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_varnibble","title":"<code>decode_varnibble(b)</code>","text":"<p>Reads first varnibble, returns its integer value and remaining bits.</p> <p>Parameters:</p> Name Type Description Default <code>b</code> <code>bitarray</code> <p>Array of header bits</p> required <p>Returns:</p> Type Description <code>Tuple[int, bitarray]</code> <p>A tuple of the integer value of first varnible and the remaining bits.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_varnibble(b):\n# type: (bitarray) -&gt; Tuple[int, bitarray]\n\"\"\"Reads first varnibble, returns its integer value and remaining bits.\n    :param bitarray b: Array of header bits\n    :return: A tuple of the integer value of first varnible and the remaining bits.\n    :rtype: Tuple[int, bitarray]\n    \"\"\"\nbits = len(b)\nif bits &gt;= 4 and b[0] == 0:\nreturn ba2int(b[:4]), b[4:]\nif bits &gt;= 8 and b[1] == 0:\nreturn ba2int(b[2:8]) + 8, b[8:]\nif bits &gt;= 12 and b[2] == 0:\nreturn ba2int(b[3:12]) + 72, b[12:]\nif bits &gt;= 16 and b[3] == 0:\nreturn ba2int(b[4:16]) + 584, b[16:]\nraise ValueError(\"Invalid bitarray\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_units","title":"<code>encode_units(units)</code>","text":"<p>Encodes a combination of ISCC units to an integer between 0-7 to be used as length value for the final encoding of MT.ISCC</p> <p>Parameters:</p> Name Type Description Default <code>units</code> <code>Tuple</code> <p>A tuple of a MainType combination (can be empty)</p> required <p>Returns:</p> Type Description <code>int</code> <p>Integer value to be used as length-value for header encoding</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_units(units):\n# type: (Tuple[MT, ...]) -&gt; int\n\"\"\"\n    Encodes a combination of ISCC units to an integer between 0-7 to be used as length\n    value for the final encoding of MT.ISCC\n    :param Tuple units: A tuple of a MainType combination (can be empty)\n    :return: Integer value to be used as length-value for header encoding\n    :rtype: int\n    \"\"\"\nreturn UNITS.index(units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_units","title":"<code>decode_units(unit_id)</code>","text":"<p>Decodes an ISCC header length value that has been encoded with a unit_id to an ordered tuple of MainTypes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_units(unit_id):\n# type: (int) -&gt; Tuple[MT, ...]\n\"\"\"\n    Decodes an ISCC header length value that has been encoded with a unit_id to an\n    ordered tuple of MainTypes.\n    \"\"\"\nunits = sorted(UNITS[unit_id])\nreturn tuple(MT(u) for u in units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_length","title":"<code>encode_length(mtype, length)</code>","text":"<p>Encode length to integer value for header encoding.</p> <p>The <code>length</code> value has MainType-specific semantics:</p> <p>For MainTypes <code>META</code>, <code>SEMANTIC</code>, <code>CONTENT</code>, <code>DATA</code>, <code>INSTANCE</code>:</p> <pre><code>Length means number of bits for the body.\nLength is encoded as the multiple of 32-bit chunks (0 being 32bits)\nExamples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n</code></pre> <p>For MainType <code>ISCC</code>:</p> <pre><code>MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\noptional. Length means the composition of optional 64-bit units included\nin the ISCC composite.\n\nExamples:\n    No optional units      -&gt; 0000 -&gt; 0\n    CONTENT                -&gt; 0001 -&gt; 1\n    SEMANTIC               -&gt; 0010 -&gt; 2\n    SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n    META                   -&gt; 0100 -&gt; 4\n    META, CONTENT          -&gt; 0101 -&gt; 5\n    ...\n</code></pre> <p>For MainType <code>ID</code>:</p> <pre><code>Lengths means number the number of bits for the body including the counter\nLength is encoded as number of bytes of the counter (64-bit body is implicit)\nExamples:\n    64 -&gt; 0 (No counter)\n    72 -&gt; 1 (One byte counter)\n    80 -&gt; 2 (Two byte counter)\n    ...\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>The MainType for which to encode the length value.</p> required <code>length</code> <code>Length</code> <p>The length expressed according to the semantics of the type</p> required <p>Returns:</p> Type Description <code>int</code> <p>The length value encoded as integer for use with write_header.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_length(mtype, length):\n# type: (MainType, Length) -&gt; int\n\"\"\"\n    Encode length to integer value for header encoding.\n    The `length` value has MainType-specific semantics:\n    For MainTypes `META`, `SEMANTIC`, `CONTENT`, `DATA`, `INSTANCE`:\n        Length means number of bits for the body.\n        Length is encoded as the multiple of 32-bit chunks (0 being 32bits)\n        Examples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n    For MainType `ISCC`:\n        MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\n        optional. Length means the composition of optional 64-bit units included\n        in the ISCC composite.\n        Examples:\n            No optional units      -&gt; 0000 -&gt; 0\n            CONTENT                -&gt; 0001 -&gt; 1\n            SEMANTIC               -&gt; 0010 -&gt; 2\n            SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n            META                   -&gt; 0100 -&gt; 4\n            META, CONTENT          -&gt; 0101 -&gt; 5\n            ...\n    For MainType `ID`:\n        Lengths means number the number of bits for the body including the counter\n        Length is encoded as number of bytes of the counter (64-bit body is implicit)\n        Examples:\n            64 -&gt; 0 (No counter)\n            72 -&gt; 1 (One byte counter)\n            80 -&gt; 2 (Two byte counter)\n            ...\n    :param MainType mtype: The MainType for which to encode the length value.\n    :param Length length: The length expressed according to the semantics of the type\n    :return: The length value encoded as integer for use with write_header.\n    :rtype: int\n    \"\"\"\nerror = f\"Invalid length {length} for MainType {mtype}\"\n# standard case (length field denotes number of 32-bit chunks, 0 being 32-bits)\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nif length &gt;= 32 and not length % 32:\nreturn (length // 32) - 1\nraise ValueError(error)\n# flag type encoding of included components (pass through as encoded out-of-band)\nelif mtype == MT.ISCC:\nif 0 &lt;= length &lt;= 7:\nreturn length\nraise ValueError(error)\n# counter byte lenght encoding\nelif mtype == MT.ID:\nif 64 &lt;= length &lt;= 96:\nreturn (length - 64) // 8\nraise ValueError(error)\nelse:\nraise ValueError(error)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_length","title":"<code>decode_length(mtype, length)</code>","text":"<p>Dedoce raw length value from ISCC header to length of digest in number of bits.</p> <p>Decodes a raw header integer value in to its semantically meaningfull value (e.g. number of bits)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_length(mtype, length):\n# type: (MainType, Length) -&gt; LN\n\"\"\"\n    Dedoce raw length value from ISCC header to length of digest in number of bits.\n    Decodes a raw header integer value in to its semantically meaningfull value (e.g.\n    number of bits)\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nreturn LN((length + 1) * 32)\nelif mtype == MT.ISCC:\nreturn LN(len(decode_units(length)) * 64 + 128)\nelif mtype == MT.ID:\nreturn LN(length * 8 + 64)\nelse:\nraise ValueError(f\"Invalid length {length} for MainType {mtype}\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32","title":"<code>encode_base32(data)</code>","text":"<p>Standard RFC4648 base32 encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base32 encoding without padding.\n    \"\"\"\nreturn b32encode(data).decode(\"ascii\").rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32","title":"<code>decode_base32(code)</code>","text":"<p>Standard RFC4648 base32 decoding without padding and with casefolding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base32 decoding without padding and with casefolding.\n    \"\"\"\n# python stdlib does not support base32 without padding, so we have to re-pad.\ncl = len(code)\npad_length = math.ceil(cl / 8) * 8 - cl\nreturn bytes(b32decode(code + \"=\" * pad_length, casefold=True))\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_decompose","title":"<code>iscc_decompose(iscc_code)</code>","text":"<p>Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.</p> <p>A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated by a hyphen.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decompose(iscc_code):\n# type: (str) -&gt; List[str]\n\"\"\"\n    Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.\n    A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated\n    by a hyphen.\n    \"\"\"\niscc_code = iscc_clean(iscc_code)\ncomponents = []\nraw_code = decode_base32(iscc_code)\nwhile raw_code:\nmt, st, vs, ln, body = decode_header(raw_code)\n# standard ISCC-UNIT with tail continuation\nif mt != MT.ISCC:\nln_bits = decode_length(mt, ln)\ncode = encode_component(mt, st, vs, ln_bits, body[: ln_bits // 8])\ncomponents.append(code)\nraw_code = body[ln_bits // 8 :]\ncontinue\n# ISCC-CODE\nmain_types = decode_units(ln)\n# rebuild dynamic units (META, SEMANTIC, CONTENT)\nfor idx, mtype in enumerate(main_types):\nstype = ST.NONE if mtype == MT.META else st\ncode = encode_component(mtype, stype, vs, 64, body[idx * 8 :])\ncomponents.append(code)\n# rebuild static units (DATA, INSTANCE)\ndata_code = encode_component(MT.DATA, ST.NONE, vs, 64, body[-16:-8])\ninstance_code = encode_component(MT.INSTANCE, ST.NONE, vs, 64, body[-8:])\ncomponents.extend([data_code, instance_code])\nbreak\nreturn components\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_normalize","title":"<code>iscc_normalize(iscc_code)</code>","text":"<p>Normalize an ISCC to its canonical form.</p> <p>The canonical form of an ISCC is its shortest base32 encoded representation prefixed with the string <code>ISCC:</code>.</p> <p>Possible valid inputs:</p> <pre><code>MEACB7X7777574L6\nISCC:MEACB7X7777574L6\nfcc010001657fe7cafe9791bb\niscc:maagztfqttvizpjr\nIscc:Maagztfqttvizpjr\n</code></pre> <p>Info</p> <p>A concatenated sequence of codes will be composed into a single ISCC of MainType <code>MT.ISCC</code> if possible.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core\n&gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Normalized ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_normalize(iscc_code):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize an ISCC to its canonical form.\n    The canonical form of an ISCC is its shortest base32 encoded representation\n    prefixed with the string `ISCC:`.\n    Possible valid inputs:\n        MEACB7X7777574L6\n        ISCC:MEACB7X7777574L6\n        fcc010001657fe7cafe9791bb\n        iscc:maagztfqttvizpjr\n        Iscc:Maagztfqttvizpjr\n    !!! info\n        A concatenated sequence of codes will be composed into a single ISCC of MainType\n        `MT.ISCC` if possible.\n    !!! example\n        ``` py\n        &gt;&gt;&gt; import iscc_core\n        &gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n        'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n        ```\n    :param str iscc_code: Any valid ISCC string\n    :return: Normalized ISCC\n    :rtype: str\n    \"\"\"\nfrom iscc_core.iscc_code import gen_iscc_code_v0\ndecoders = {\nMULTIBASE.base16.value: bytes.fromhex,  # f\nMULTIBASE.base32.value: decode_base32,  # b\nMULTIBASE.base32hex.value: decode_base32hex,  # v\nMULTIBASE.base58btc.value: base58.b58decode,  # z\nMULTIBASE.base64url.value: decode_base64,  # u\n}\n# Transcode to base32 if &lt;multibase&gt;&lt;multicodec&gt; encoded\nmultibase_prefix = iscc_code[0]\nif multibase_prefix in decoders.keys():\ndecoder = decoders[multibase_prefix]\ndecoded = decoder(iscc_code[1:])\nif not decoded.startswith(MC_PREFIX):\nraise ValueError(f\"Malformed multiformat codec: {decoded[:2]}\")\niscc_code = encode_base32(decoded[2:])\nelse:\nprefix = iscc_code.lstrip(\"ISCC:\").lstrip(\"iscc:\")[:2].upper()\nif prefix not in PREFIXES:\nraise ValueError(f\"ISCC starts with invalid prefix {prefix}\")\ndecomposed = iscc_decompose(iscc_code)\nrecomposed = gen_iscc_code_v0(decomposed)[\"iscc\"] if len(decomposed) &gt;= 2 else decomposed[0]\nreturn f\"ISCC:{recomposed}\" if not recomposed.startswith(\"ISCC:\") else recomposed\n</code></pre>"},{"location":"codec/#alternate-encodings","title":"Alternate Encodings","text":""},{"location":"codec/#iscc_core.codec.encode_base64","title":"<code>encode_base64(data)</code>","text":"<p>Standard RFC4648 base64url encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base64(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base64url encoding without padding.\n    \"\"\"\ncode = urlsafe_b64encode(data).decode(\"ascii\")\nreturn code.rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base64","title":"<code>decode_base64(code)</code>","text":"<p>Standard RFC4648 base64url decoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base64(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base64url decoding without padding.\n    \"\"\"\npadding = 4 - (len(code) % 4)\nstring = code + (\"=\" * padding)\nreturn urlsafe_b64decode(string)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32hex","title":"<code>encode_base32hex(data)</code>","text":"<p>RFC4648 Base32hex encoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32hex(data):\n# type: (bytes) -&gt;  str\n\"\"\"\n    RFC4648 Base32hex encoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = encode_base32(data)\nreturn b32.translate(b32_to_hex)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32hex","title":"<code>decode_base32hex(code)</code>","text":"<p>RFC4648 Base32hex decoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32hex(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    RFC4648 Base32hex decoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = code.translate(hex_to_b32)\nreturn decode_base32(b32)\n</code></pre>"},{"location":"codec/#helper-functions","title":"Helper Functions","text":""},{"location":"codec/#iscc_core.codec.iscc_decode","title":"<code>iscc_decode(iscc)</code>","text":"<p>Decode ISCC to an IsccTuple</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>ISCC decoded to a tuple</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decode(iscc):\n# type: (str) -&gt; IsccTuple\n\"\"\"\n    Decode ISCC to an IsccTuple\n    :param str iscc: ISCC string\n    :return: ISCC decoded to a tuple\n    :rtype: IsccTuple\n    \"\"\"\niscc = iscc_clean(iscc_normalize(iscc))\ndata = decode_base32(iscc)\nreturn decode_header(data)\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_explain","title":"<code>iscc_explain(iscc)</code>","text":"<p>Convert ISCC to a human-readable representation</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Human-readable representation of ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_explain(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Convert ISCC to a human-readable representation\n    :param str iscc: ISCC string\n    :return: Human-readable representation of ISCC\n    :rtype: str\n    \"\"\"\ntid = iscc_type_id(iscc)\nfields = iscc_decode(iscc)\nif fields[0] == MT.ID:\ncounter_bytes = fields[-1][8:]\nif counter_bytes:\ncounter = uvarint.decode(counter_bytes)\nhex_hash = fields[-1][:8].hex()\nreturn f\"{tid}-{hex_hash}-{counter.integer}\"\nhex_hash = fields[-1].hex()\nreturn f\"{tid}-{hex_hash}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_type_id","title":"<code>iscc_type_id(iscc)</code>","text":"<p>Extract and convert ISCC HEADER to a readable Type-ID string.</p> <p>Type-ids can be used as names in databases to index ISCC-UNITs seperatly.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Unique Type-ID string</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_type_id(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Extract and convert ISCC HEADER to a readable Type-ID string.\n    Type-ids can be used as names in databases to index ISCC-UNITs seperatly.\n    :param str iscc: ISCC string\n    :return: Unique Type-ID string\n    :rtype: str\n    \"\"\"\nfields = iscc_decode(iscc)\nmtype = MT(fields[0])\nstype = SUBTYPE_MAP[fields[0]](fields[1])\nif mtype == MT.ISCC:\nmtypes = decode_units(fields[3])\nlength = \"\".join([t.name[0] for t in mtypes]) + \"DI\"\nelse:\nlength = decode_length(fields[0], fields[3])\nversion = VS(fields[2])\nreturn f\"{mtype.name}-{stype.name}-{version.name}-{length}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_validate","title":"<code>iscc_validate(iscc, strict = True)</code>","text":"<p>Validate that a given string is a strictly well-formed ISCC.</p> <p>A strictly well-formed ISCC is:</p> <ul> <li>an ISCC-CODE or ISCC-UNIT</li> <li>encoded with base32 upper without padding</li> <li>has a valid combination of header values</li> <li>is represented in its canonical URI form</li> </ul> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <code>strict</code> <code>bool</code> <p>Raise an exeption if validation fails (default True)</p> <code>True</code> <p>Returns:</p> Type Description <code>bool</code> <p>True if sting is valid else false. (raises ValueError in strict mode)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_validate(iscc, strict=True):\n# type: (str, bool) -&gt; bool\n\"\"\"\n    Validate that a given string is a *strictly well-formed* ISCC.\n    A *strictly well-formed* ISCC is:\n    - an ISCC-CODE or ISCC-UNIT\n    - encoded with base32 upper without padding\n    - has a valid combination of header values\n    - is represented in its canonical URI form\n    :param str iscc: ISCC string\n    :param bool strict: Raise an exeption if validation fails (default True)\n    :return: True if sting is valid else false. (raises ValueError in strict mode)\n    :rtype: bool\n    \"\"\"\n# Basic regex validation\nmatch = re.match(\"^ISCC:[A-Z2-7]{10,60}$\", iscc)\nif not match:\nif strict:\nraise ValueError(\"ISCC string does not match ^ISCC:[A-Z2-7]{10,60}$\")\nelse:\nreturn False\ncleaned = iscc_clean(iscc)\nprefix = cleaned[:2]\nif prefix not in PREFIXES:\nif strict:\nraise ValueError(f\"Header starts with invalid sequence {prefix}\")\nelse:\nreturn False\nm, s, v, l, t = decode_header(decode_base32(cleaned))\nif v != 0:\nif strict:\nraise ValueError(f\"Unknown version {v} in version header\")\nelse:\nreturn False\nreturn True\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_clean","title":"<code>iscc_clean(iscc)</code>","text":"<p>Cleanup ISCC string.</p> <p>Removes leading scheme, dashes, leading/trailing whitespace.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Cleaned ISCC string.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_clean(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Cleanup ISCC string.\n    Removes leading scheme, dashes, leading/trailing whitespace.\n    :param str iscc: Any valid ISCC string\n    :return: Cleaned ISCC string.\n    :rtype: str\n    \"\"\"\nsplit = [part.strip() for part in iscc.strip().split(\":\")]\nif len(split) == 1:\ncode = split[0]\n# remove dashes if not multiformat\nif code[0] not in list(MULTIBASE):\ncode = code.replace(\"-\", \"\")\nreturn code\nelif len(split) == 2:\nscheme, code = split\nif scheme.lower() != \"iscc\":\nraise ValueError(f\"Invalid scheme: {scheme}\")\nreturn code.replace(\"-\", \"\")\nelse:\nraise ValueError(f\"Malformed ISCC string: {iscc}\")\n</code></pre>"},{"location":"options/options/","title":"ISCC-CORE - Configuration Options","text":"<p>Options for the iscc-core package can be configured using environment variables. Variables are loaded as class-attributes on the <code>CoreOptions</code> instance. Environment variables are named like the class-attribute but prefixed with <code>ISCC_CORE_</code> and upper-cased.</p> <p>Example how to access configuration options</p> <pre><code>import iscc_core as ic\n# To access ISCC_CORE_TEXT_NGRAM_SIZE setting use\ntext_ngram_size: int = ic.core_opts.text_ngram_size\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions","title":"CoreOptions","text":"<p>Parameters with defaults for ISCC calculations.</p>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_bits","title":"meta_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_bits: int = Field(\n64,\ndescription=\"Default length of generated Meta-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_name","title":"meta_trim_name  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_trim_name: int = Field(\n128, description=\"Trim `name` to this mumber of bytes\"\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_description","title":"meta_trim_description  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_trim_description: int = Field(\n4096,\ndescription=\"Trim `description` to this number of bytes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_text","title":"meta_ngram_size_text  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_ngram_size_text: int = Field(\n3,\ndescription=\"Sliding window width (characters) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_bytes","title":"meta_ngram_size_bytes  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_ngram_size_bytes: int = Field(\n4,\ndescription=\"Sliding window width (bytes) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_bits","title":"text_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Text in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_ngram_size","title":"text_ngram_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_ngram_size: int = Field(\n13,\ndescription=\"Number of characters per feature hash (size of sliding window)\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_unicode_filter","title":"text_unicode_filter  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_unicode_filter: frozenset = Field(\nfrozenset({\"C\", \"M\", \"P\"}),\ndescription=\"Unicode categories to remove during text normalization\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_newlines","title":"text_newlines  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_newlines: frozenset = Field(\nfrozenset(\n{\n\"\\n\",\n\"\\x0b\",\n\"\\x0c\",\n\"\\r\",\n\"\\x85\",\n\"\\u2028\",\n\"\\u2029\",\n}\n),\ndescription=\"Characters regarded as newline characters for normalization purposes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.image_bits","title":"image_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>image_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Image in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.audio_bits","title":"audio_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>audio_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Audio in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.video_bits","title":"video_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>video_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Video in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_bits","title":"data_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>data_bits: int = Field(\n64,\ndescription=\"Default length of generated Data-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_avg_chunk_size","title":"data_avg_chunk_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>data_avg_chunk_size: int = Field(\n1024,\ndescription=\"Target chunk size for data chunking in number of bytes.\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.instance_bits","title":"instance_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>instance_bits: int = Field(\n64,\ndescription=\"Default length of generated Instance-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.mixed_bits","title":"mixed_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>mixed_bits: int = Field(\n64,\ndescription=\"Default length of generated Mixed-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.io_read_size","title":"io_read_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>io_read_size: int = Field(\n2097152,\ndescription=\"File read buffer size in bytes for hashing operations\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.cdc_gear","title":"cdc_gear  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>cdc_gear: Tuple = Field(\n(\n1553318008,\n574654857,\n759734804,\n310648967,\n1393527547,\n1195718329,\n694400241,\n1154184075,\n1319583805,\n1298164590,\n122602963,\n989043992,\n1918895050,\n933636724,\n1369634190,\n1963341198,\n1565176104,\n1296753019,\n1105746212,\n1191982839,\n1195494369,\n29065008,\n1635524067,\n722221599,\n1355059059,\n564669751,\n1620421856,\n1100048288,\n1018120624,\n1087284781,\n1723604070,\n1415454125,\n737834957,\n1854265892,\n1605418437,\n1697446953,\n973791659,\n674750707,\n1669838606,\n320299026,\n1130545851,\n1725494449,\n939321396,\n748475270,\n554975894,\n1651665064,\n1695413559,\n671470969,\n992078781,\n1935142196,\n1062778243,\n1901125066,\n1935811166,\n1644847216,\n744420649,\n2068980838,\n1988851904,\n1263854878,\n1979320293,\n111370182,\n817303588,\n478553825,\n694867320,\n685227566,\n345022554,\n2095989693,\n1770739427,\n165413158,\n1322704750,\n46251975,\n710520147,\n700507188,\n2104251000,\n1350123687,\n1593227923,\n1756802846,\n1179873910,\n1629210470,\n358373501,\n807118919,\n751426983,\n172199468,\n174707988,\n1951167187,\n1328704411,\n2129871494,\n1242495143,\n1793093310,\n1721521010,\n306195915,\n1609230749,\n1992815783,\n1790818204,\n234528824,\n551692332,\n1930351755,\n110996527,\n378457918,\n638641695,\n743517326,\n368806918,\n1583529078,\n1767199029,\n182158924,\n1114175764,\n882553770,\n552467890,\n1366456705,\n934589400,\n1574008098,\n1798094820,\n1548210079,\n821697741,\n601807702,\n332526858,\n1693310695,\n136360183,\n1189114632,\n506273277,\n397438002,\n620771032,\n676183860,\n1747529440,\n909035644,\n142389739,\n1991534368,\n272707803,\n1905681287,\n1210958911,\n596176677,\n1380009185,\n1153270606,\n1150188963,\n1067903737,\n1020928348,\n978324723,\n962376754,\n1368724127,\n1133797255,\n1367747748,\n1458212849,\n537933020,\n1295159285,\n2104731913,\n1647629177,\n1691336604,\n922114202,\n170715530,\n1608833393,\n62657989,\n1140989235,\n381784875,\n928003604,\n449509021,\n1057208185,\n1239816707,\n525522922,\n476962140,\n102897870,\n132620570,\n419788154,\n2095057491,\n1240747817,\n1271689397,\n973007445,\n1380110056,\n1021668229,\n12064370,\n1186917580,\n1017163094,\n597085928,\n2018803520,\n1795688603,\n1722115921,\n2015264326,\n506263638,\n1002517905,\n1229603330,\n1376031959,\n763839898,\n1970623926,\n1109937345,\n524780807,\n1976131071,\n905940439,\n1313298413,\n772929676,\n1578848328,\n1108240025,\n577439381,\n1293318580,\n1512203375,\n371003697,\n308046041,\n320070446,\n1252546340,\n568098497,\n1341794814,\n1922466690,\n480833267,\n1060838440,\n969079660,\n1836468543,\n2049091118,\n2023431210,\n383830867,\n2112679659,\n231203270,\n1551220541,\n1377927987,\n275637462,\n2110145570,\n1700335604,\n738389040,\n1688841319,\n1506456297,\n1243730675,\n258043479,\n599084776,\n41093802,\n792486733,\n1897397356,\n28077829,\n1520357900,\n361516586,\n1119263216,\n209458355,\n45979201,\n363681532,\n477245280,\n2107748241,\n601938891,\n244572459,\n1689418013,\n1141711990,\n1485744349,\n1181066840,\n1950794776,\n410494836,\n1445347454,\n2137242950,\n852679640,\n1014566730,\n1999335993,\n1871390758,\n1736439305,\n231222289,\n603972436,\n783045542,\n370384393,\n184356284,\n709706295,\n1453549767,\n591603172,\n768512391,\n854125182,\n),\ndescription=\"Random gear vector\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformanc_critical","title":"conformanc_critical  <code>module-attribute</code>","text":"<pre><code>conformanc_critical = {\n\"meta_trim_name\",\n\"meta_trim_description\",\n\"meta_ngram_size_text\",\n\"meta_ngram_size_bytes\",\n\"text_ngram_size\",\n\"text_unicode_filter\",\n\"text_newlines\",\n\"data_avg_chunk_size\",\n\"cdc_gear\",\n}\n</code></pre>"},{"location":"options/options/#iscc_core.options.has_logged_confromance","title":"has_logged_confromance  <code>module-attribute</code>","text":"<pre><code>has_logged_confromance = False\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformance_check_options","title":"conformance_check_options","text":"<pre><code>conformance_check_options(opts)\n</code></pre> <p>Check and log if options have non-default conformance critical values</p>"},{"location":"options/options/#iscc_core.options.core_opts","title":"core_opts  <code>module-attribute</code>","text":"<pre><code>core_opts = CoreOptions()\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformant_options","title":"conformant_options  <code>module-attribute</code>","text":"<pre><code>conformant_options = conformance_check_options(core_opts)\n</code></pre>"},{"location":"units/","title":"ISCC - UNITs","text":"<p>A standard ISCC-CODE is build from multiple ISCC-UNITs. Each unit serve a different purpose.</p>"},{"location":"units/code_data/","title":"ISCC - Data-Code","text":"<p>A similarity perserving hash for binary data (soft hash).</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code","title":"<code>gen_data_code(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create a similarity preserving ISCC Data-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code_v0","title":"<code>gen_data_code_v0(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.soft_hash_data_v0","title":"<code>soft_hash_data_v0(stream)</code>","text":"<p>Create a similarity preserving Data-Hash digest</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Data-Hash (soft-hash) digest used as body for Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0","title":"<code>DataHasherV0</code>","text":"<p>Incremental Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.__init__","title":"<code>__init__(data = None)</code>","text":"<p>Create a DataHasher</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Optional[Data]</code> <p>initial payload for hashing.</p> <code>None</code>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.digest","title":"<code>digest()</code>","text":"<p>Calculate 256-bit minhash digest from feature hashes.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.code","title":"<code>code(bits = ic.core_opts.data_bits)</code>","text":"<p>Encode digest as an ISCC Data-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Data-Code</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Data-Code</p>"},{"location":"units/code_flake/","title":"ISCC - Flake-Code","text":"<p>A unique, time-sorted identifier composed of an 48-bit timestamp and 16 to 208 bit randomness.</p> <p>The ISCC Flake-Code is a unique identifier for distributed ID generation. The 64-bit version can be used as efficient surrogate key in database systems. It has guaranteed uniqueness if generated from a singele process and is time sortable in integer and base32hex representation. The 128-bit version is a K-sortable, globally unique identifier for use in distributed systems and is compatible with UUID.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core as ic\n&gt;&gt;&gt; ic.gen_flake_code(bits=64)\n{'iscc': 'ISCC:OAAQC7YN7PG2XOR4'}\n&gt;&gt;&gt; ic.gen_flake_code(bits=128)\n{'iscc': 'ISCC:OABQC7YN7RJGUUTLKDSKBXO25MA5E'}\n# Or use the convenience Flake class for easy access to different representations\n&gt;&gt;&gt; flake = ic.Flake(bits=64)\n&gt;&gt;&gt; flake.iscc\n'ISCC:OAAQC7YOADBZYNF7'\n&gt;&gt;&gt; flake.time\n'2022-02-18T18:03:25.468'\n&gt;&gt;&gt; flake.int\n107820312524764351\n&gt;&gt;&gt; flake.string\n'05VGS063JGQBU'\n</code></pre>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code","title":"<code>gen_flake_code(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest standard algorithm</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code_v0","title":"<code>gen_flake_code_v0(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest algorithm v0</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.uid_flake_v0","title":"<code>uid_flake_v0(ts = None, bits = ic.core_opts.flake_bits)</code>","text":"<p>Generate time and randomness based Flake-Hash</p> <p>Parameters:</p> Name Type Description Default <code>ts</code> <code>Optional[float]</code> <p>Unix timestamp (defaults to current time)</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length resulting Flake-Code (multiple of 32)</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Flake-Hash digest</p>"},{"location":"units/code_instance/","title":"ISCC - Instance-Code","text":"<p>A data checksum.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code","title":"<code>gen_instance_code(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with properties: iscc, datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code_v0","title":"<code>gen_instance_code_v0(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.hash_instance_v0","title":"<code>hash_instance_v0(stream)</code>","text":"<p>Create 256-bit hash digest for the Instance-Code body</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for hash generation.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Instance-Hash digest used as body of Instance-Code</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0","title":"<code>InstanceHasherV0</code>","text":"<p>Incremental Instance-Hash generator.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Instance-Hash generator.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Data</code> <p>Data to be hashed</p> required"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.digest","title":"<code>digest()</code>","text":"<p>Return Instance-Hash</p> <p>Returns:</p> Type Description <code>bytes</code> <p>Instance-Hash digest</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.multihash","title":"<code>multihash()</code>","text":"<p>Return blake3 multihash</p> <p>Returns:</p> Type Description <code>str</code> <p>Blake3 hash as 256-bit multihash</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.code","title":"<code>code(bits = ic.core_opts.instance_bits)</code>","text":"<p>Encode digest as an ISCC Instance-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Instance-Code</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Instance-Code</p>"},{"location":"units/code_meta/","title":"ISCC - Meta-Code","text":"<p>A similarity preserving hash for digital asset metadata.</p>"},{"location":"units/code_meta/#purpose","title":"Purpose","text":"<p>The Meta-Code is the first possible (optional) unit of an ISCC-CODE. It is calculated from the metadata of a digital asset. The primary purpose of the Meta-Code is to aid the discovery of digital assets with similar metadata and the detection of metadata anomalies. As a secondary function, Meta-Code processing also creates a secure Meta-Hash for cryptogrpahic binding purposes.</p>"},{"location":"units/code_meta/#inputs","title":"Inputs","text":"<p>The metadata supplied for Meta-Code calculation is called Seed-Metadata. Seed-Metadata has 3 possible elements:</p> <ul> <li>name (required): The name or title of the work manifested by the digital asset.</li> <li>description (optional): A disambiguating textual description of the digital asset.</li> <li>meta (optional): Industry-sector or use-case specific metadata, encoded as     Data-URL (RFC-2397).</li> </ul> <p>Note</p> <p>Due to the broad applicability of the ISCC we do not prescribe a particular schema for the <code>meta</code>-element. Instead we use the Data-URL format because it can encode and self-describe any conceivable metadata in a sufficently machine-interpretable form at any desired specificity.</p> <p>Data-URL Examples:</p> <ul> <li>Metadata is \"some\" JSON: <code>data:application/json;base64,&lt;data&gt;</code></li> <li>Metadata is JSON-LD: <code>data:application/ld+json;base64,&lt;data&gt;</code></li> <li>Metadata is \"some\" XML: <code>data:application/xml;base64,&lt;data&gt;</code></li> <li>Metadata is MARC21 XML: <code>data:application/marcxml+xml;base64,&lt;data&gt;</code></li> <li>Metadata is IPTC NewsML: <code>data:application/vnd.iptc.g2.newsitem+xml;base64,&lt;data&gt;</code></li> </ul> <p>Data-URLs are also supported by all major internet browsers.</p>"},{"location":"units/code_meta/#processing","title":"Processing","text":""},{"location":"units/code_meta/#meta-code","title":"Meta-Code","text":"<p>The first 32-bits of a Meta-Code are calculated as a simliarity hash from the <code>name</code> field. The second 32-bits are also calculated from the <code>name</code> field if no other input was supplied. If <code>description</code> is suplied but no <code>meta</code>, the <code>description</code> will be used for the second 32-bits. If <code>meta</code> is supplied it will be used in favour of <code>description</code> for the second 32-bits.</p> <pre><code>flowchart LR\n    B{name?}\n    B --&gt;|Yes| J[\"P1 = SH(name)\"]\n    J --&gt; D{meta?}\n    D --&gt;|Yes| F[\"Meta-Code = P1 + SH(meta)\"]\n    D --&gt;|No| G{description?}\n    G --&gt;|Yes| H[\"Meta-Code = P1 + SH(description)\"]\n    G --&gt;|No| I[\"Meta-Code = P1\"]\n    B --&gt;|No| E[Skip Meta-Code]</code></pre> <p>Note</p> <p>To support automation and reproducibility, applications that generate ISCCs, should prioritize metadata that is automatically extracted from the digital asset.</p> <p>If embedded metadata is not available or known to be unreliable an application should rely on external metadata or explicitly ask users to supply at least the <code>name</code>-field. Applications should then first embed metadata into the asset before calculating the ISCC-CODE. This ensures that the embedded metadata is bound to the asset by the Instance-Code.</p> <p>If neither embedded nor external metadata is available, the application may resort to use the filename of the digital asset as value for the <code>name</code>-field. If no value can be determined for the <code>name</code>-field, an application shall skip generation of a Meta-Code and create an ISCC-CODE without a Meta-Code.</p>"},{"location":"units/code_meta/#meta-hash","title":"Meta-Hash","text":"<p>In addition to the Meta-Code we also create a cryptographic hash (the Meta-Hash) of the supplied Seed-Metadata. It is used to securely bind metadata to the digital asset.</p> <pre><code>flowchart LR\n    N{name?}\n    N --&gt;|Yes| A{meta?}\n    N --&gt;|No| E[Skip Meta-Hash]\n    A --&gt;|Yes| H[\"Meta-Hash = H(meta)\"]\n    A --&gt;|No| B{description?}\n    B --&gt;|Yes| HND[\"Meta-Hash = H(name + description)\"]\n    B --&gt;|No| HN[\"Meta-Hash = H(name)\"]</code></pre>"},{"location":"units/code_meta/#functions","title":"Functions","text":""},{"location":"units/code_meta/#iscc_core.code_meta.gen_meta_code_v0","title":"<code>gen_meta_code_v0(name, description = None, meta = None, bits = ic.core_opts.meta_bits)</code>","text":"<p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.soft_hash_meta_v0","title":"<code>soft_hash_meta_v0(name, extra = None)</code>","text":"<p>Calculate simmilarity preserving 256-bit hash digest from asset metadata.</p> <p>Textual input should be stripped of markup, normalized and trimmed before hashing. Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be serialized in a canonical form (for example JCS for JSON)</p> <p>Note</p> <p>The processing algorithm depends on the type of the <code>extra</code> input. If the <code>extra</code> field is supplied and non-empty, we create separate hashes for <code>name</code> and <code>extra</code> and interleave them in 32-bit chunks:</p> <ul> <li> <p>If the <code>extra</code> input is <code>None</code> or an empty <code>str</code>/<code>bytes</code> object the Meta-Hash will be generated from the <code>name</code>-field only.</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty text string (str) the string is lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty bytes object the processing is done bytewise.</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Title of the work manifested in the digital asset</p> required <code>extra</code> <code>Union[str,bytes,None]</code> <p>Additional metadata for disambiguation</p> <code>None</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit simhash digest for Meta-Code</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def soft_hash_meta_v0(name, extra=None):\n# type: (str, Union[str,bytes,None]) -&gt; bytes\n\"\"\"\n    Calculate simmilarity preserving 256-bit hash digest from asset metadata.\n    Textual input should be stripped of markup, normalized and trimmed before hashing.\n    Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be\n    serialized in a canonical form (for example\n    [JCS](https://tools.ietf.org/id/draft-rundgren-json-canonicalization-scheme-00.html) for JSON)\n    !!! note\n        The processing algorithm depends on the type of the `extra` input.\n        If the `extra` field is supplied and non-empty, we create separate hashes for\n        `name` and `extra` and interleave them in 32-bit chunks:\n        - If the `extra` input is `None` or an empty `str`/`bytes` object the Meta-Hash will\n        be generated from the `name`-field only.\n        - If the `extra`-input is a non-empty **text** string (str) the string is\n        lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).\n        - If the `extra`-input is a non-empty **bytes** object the processing is done bytewise.\n    :param str name: Title of the work manifested in the digital asset\n    :param Union[str,bytes,None] extra: Additional metadata for disambiguation\n    :return: 256-bit simhash digest for Meta-Code\n    :rtype: bytes\n    \"\"\"\nname = ic.text_collapse(name)\nname_n_grams = ic.sliding_window(name, width=ic.core_opts.meta_ngram_size_text)\nname_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in name_n_grams]\nsimhash_digest = ic.alg_simhash(name_hash_digests)\nif extra in {None, \"\", b\"\"}:\nreturn simhash_digest\nelse:\n# Augment with interleaved hash for extra metadata\nif isinstance(extra, bytes):\n# Raw bytes are handled per byte\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_bytes)\nextra_hash_digests = [blake3(ngram).digest() for ngram in extra_n_grams]\nelif isinstance(extra, str):\n# Text is collapsed and handled per character (multibyte)\nextra = ic.text_collapse(extra)\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_text)\nextra_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in extra_n_grams]\nelse:\nraise ValueError(\"parameter `extra` must be of type str or bytes!\")\nextra_simhash_digest = ic.alg_simhash(extra_hash_digests)\n# Interleave first half of name and extra simhashes in 32-bit chunks\nchunks_simhash_digest = sliced(simhash_digest[:16], 4)\nchunks_extra_simhash_digest = sliced(extra_simhash_digest[:16], 4)\ninterleaved = interleave(chunks_simhash_digest, chunks_extra_simhash_digest)\nsimhash_digest = bytearray()\nfor chunk in interleaved:\nsimhash_digest += chunk\nsimhash_digest = bytes(simhash_digest)\nreturn simhash_digest\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_clean","title":"<code>text_clean(text)</code>","text":"<p>Clean text for display.</p> <ul> <li>Normalize with NFKC normalization.</li> <li>Remove Control Characters (except newlines)</li> <li>Reduce multiple consecutive newlines to a maximum of two newlines</li> <li>Strip leading and trailing whitespace</li> </ul> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_clean(text):\n# type: (str) -&gt; str\n\"\"\"\n    Clean text for display.\n    - Normalize with NFKC normalization.\n    - Remove Control Characters (except newlines)\n    - Reduce multiple consecutive newlines to a maximum of two newlines\n    - Strip leading and trailing whitespace\n    \"\"\"\n# Unicode normalize\ntext = unicodedata.normalize(\"NFKC\", text)\n# Remove control characters\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] != \"C\" or ch in ic.core_opts.text_newlines\n)\n# Collapse more than two consecutive newlines\nchars = []\nnewline_count = 0\nfor c in text:\nif c in ic.core_opts.text_newlines:\nif newline_count &lt; 2:\nchars.append(\"\\n\")\nnewline_count += 1\ncontinue\nelse:\nnewline_count = 0\nchars.append(c)\ntext = \"\".join(chars)\nreturn text.strip()\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_remove_newlines","title":"<code>text_remove_newlines(text)</code>","text":"<p>Remove newlines.</p> <p>The <code>name</code> field serves as a displayable title. We remove newlines and leading and trailing whitespace. We also collapse consecutive spaces to single spaces.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <p>Text for newline removal</p> required <p>Returns:</p> Type Description <code>str</code> <p>Single line of text</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_remove_newlines(text):\n# type: (str) -&gt; str\n\"\"\"\n    Remove newlines.\n    The `name` field serves as a displayable title. We remove newlines and leading and trailing\n    whitespace. We also collapse consecutive spaces to single spaces.\n    :param text: Text for newline removal\n    :return: Single line of text\n    :rtype: str\n    \"\"\"\nreturn \" \".join(text.split())\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_trim","title":"<code>text_trim(text, nbytes)</code>","text":"<p>Trim text such that its utf-8 encoded size does not exceed <code>nbytes</code>.</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_trim(text, nbytes):\n# type: (str, int) -&gt; str\n\"\"\"Trim text such that its utf-8 encoded size does not exceed `nbytes`.\"\"\"\nreturn text.encode(\"utf-8\")[:nbytes].decode(\"utf-8\", \"ignore\").strip()\n</code></pre>"},{"location":"units/content/","title":"ISCC - Content-Codes","text":""},{"location":"units/content/code_content_audio/","title":"ISCC - Audio-Code","text":"<p>A similarity preserving hash for audio content (soft hash).</p> <p>Creates an ISCC object that provides an <code>iscc</code>-field with an Audio-Code and a <code>duration</code>-field.</p> <p>The Content-Code Audio is generated from a Chromaprint fingerprint provided as a vector of 32-bit signed integers. The iscc-sdk uses fpcalc to extract Chromaprint vectors with the following command line parameters:</p> <p><code>$ fpcalc -raw -json -signed -length 0 myaudiofile.mp3</code></p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code","title":"<code>gen_audio_code(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code_v0","title":"<code>gen_audio_code_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.soft_hash_audio_v0","title":"<code>soft_hash_audio_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create audio similarity hash from a chromaprint vector.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting similarity hash (multiple of 32)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Audio-Hash digest</p>"},{"location":"units/content/code_content_image/","title":"ISCC - Image-Code","text":"<p>A similarity preserving perceptual hash for images.</p> <p>The ISCC Content-Code Image is created by calculating a discrete cosine transform on normalized image-pixels and comparing the values from the upper left area of the dct-matrix against their median values to set the hash-bits.</p> <p>Images must be normalized before using gen_image_code. Prepare images as follows:</p> <ul> <li>Transpose image according to EXIF Orientation</li> <li>Add white background to image if it has alpha transparency</li> <li>Crop empty borders of image</li> <li>Convert image to grayscale</li> <li>Resize image to 32x32</li> <li>Flatten 32x32 matrix to an array of 1024 grayscale (uint8) pixel values</li> </ul>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code","title":"<code>gen_image_code(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values).</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code_v0","title":"<code>gen_image_code_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.soft_hash_image_v0","title":"<code>soft_hash_image_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Calculate image hash from normalized grayscale pixel sequence of length 1024.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> required <code>bits</code> <code>int</code> <p>Bit-length of image hash (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving Image-Hash digest.</p>"},{"location":"units/content/code_content_mixed/","title":"ISCC - Mixed Code","text":"<p>A similarity hash for mixed media content.</p> <p>Creates an ISCC object that provides a <code>iscc</code>-field a Mixed-Code and a <code>parts</code>-field that lists the input codes.</p> <p>Many digital assets embed multiple assets of different mediatypes in a single file. Text documents may include images, video includes audio in most cases. The ISCC Content-Code-Mixed encodes the similarity of a collection of assets of the same or different mediatypes that may occur in a multimedia asset.</p> <p>Applications that create mixed Content-Codes must be capable to extract embedded assets and create individual Content-Codes per asset.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code","title":"<code>gen_mixed_code(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code Mixed with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code_v0","title":"<code>gen_mixed_code_v0(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.soft_hash_codes_v0","title":"<code>soft_hash_codes_v0(cc_digests, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create a similarity hash from multiple Content-Code digests.</p> <p>The similarity hash is created from the bodies of the input codes with the first byte of the code-header prepended.</p> <p>All codes must be of main-type CONTENT and have a minimum length of <code>bits</code>.</p> <p>Parameters:</p> Name Type Description Default <code>cc_digests</code> <code>Sequence[bytes]</code> <p>a list of Content-Code digests.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving byte hash.</p>"},{"location":"units/content/code_content_text/","title":"ISCC - Text Code","text":"<p>A similarity preserving hash for plain-text content (soft hash).</p> <p>The ISCC Text-Code is generated from plain-text that has been extracted from a media assets.</p> <p>Warning</p> <p>Plain-text extraction from documents in various formats (especially PDF) may yield very diffent results depending on the extraction tools being used. The iscc-sdk uses Apache Tika to extract text from documents for Text-Code generation.</p> <p>Algorithm overview</p> <ul> <li>Apply <code>text_collapse</code> function to text input</li> <li>Count characters of collapsed text</li> <li>Apply <code>soft_hash_text_v0</code> to collapsed text</li> </ul>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.gen_text_code_v0","title":"<code>gen_text_code_v0(text, bits = ic.core_opts.text_bits)</code>","text":"<p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.text_collapse","title":"<code>text_collapse(text)</code>","text":"<p>Normalize and simplify text for similarity hashing.</p> <ul> <li>Decompose with NFD normalization.</li> <li>Remove all whitespace characters and convert text to lower case</li> <li>Filter control characters, marks (diacritics), and punctuation</li> <li>Recombine with NFKC normalization.</li> </ul> <p>Note</p> <p>See: Unicode normalization.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be collapsed.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Collapsed plain text.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def text_collapse(text):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize and simplify text for similarity hashing.\n    - Decompose with NFD normalization.\n    - Remove all whitespace characters and convert text to lower case\n    - Filter control characters, marks (diacritics), and punctuation\n    - Recombine with NFKC normalization.\n    !!! note\n        See: [Unicode normalization](https://unicode.org/reports/tr15/).\n    :param str text: Plain text to be collapsed.\n    :return: Collapsed plain text.\n    :rtype: str\n    \"\"\"\n# Decompose with NFD\ntext = unicodedata.normalize(\"NFD\", text)\n# Remove all whitespace and convert text to lower case\ntext = \"\".join(text.split()).lower()\n# Filter control characters, marks (diacritics), and punctuation\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] not in ic.core_opts.text_unicode_filter\n)\n# Recombine\ntext = unicodedata.normalize(\"NFKC\", text)\nreturn text\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.soft_hash_text_v0","title":"<code>soft_hash_text_v0(text)</code>","text":"<p>Creates a 256-bit similarity preserving hash for text input with algorithm v0.</p> <ul> <li>Slide over text with a   <code>text_ngram_size</code> wide window   and create <code>xxh32</code> hashes</li> <li>Create a <code>minhash_256</code> from the hashes generated   in the previous step.</li> </ul> <p>Note</p> <p>Before passing text to this function it must be:</p> <ul> <li>stripped of markup</li> <li>normalized</li> <li>stripped of whitespace</li> <li>lowercased</li> </ul> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be hashed.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit similarity preserving byte hash.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def soft_hash_text_v0(text):\n# type: (str) -&gt; bytes\n\"\"\"\n    Creates a 256-bit similarity preserving hash for text input with algorithm v0.\n    - Slide over text with a\n      [`text_ngram_size`][iscc_core.options.CoreOptions.text_ngram_size] wide window\n      and create [`xxh32`](https://cyan4973.github.io/xxHash/) hashes\n    - Create a [`minhash_256`][iscc_core.minhash.alg_minhash_256] from the hashes generated\n      in the previous step.\n    !!! note\n        Before passing text to this function it must be:\n        - stripped of markup\n        - normalized\n        - stripped of whitespace\n        - lowercased\n    :param str text: Plain text to be hashed.\n    :return: 256-bit similarity preserving byte hash.\n    :rtype: bytes\n    \"\"\"\nngrams = ic.sliding_window(text, ic.core_opts.text_ngram_size)\nfeatures = [xxhash.xxh32_intdigest(s.encode(\"utf-8\")) for s in ngrams]\nhash_digest = ic.alg_minhash_256(features)\nreturn hash_digest\n</code></pre>"},{"location":"units/content/code_content_video/","title":"ISCC - Video-Code","text":"<p>A similarity preserving hash for video content</p> <p>The Content-Code Video is generated from MPEG-7 video frame signatures. The iscc-sdk uses ffmpeg to extract frame signatures with the following command line parameters:</p> <p><code>$ ffmpeg -i video.mpg -vf fps=fps=5,signature=format=xml:filename=sig.xml -f null -</code></p> <p>The relevant frame signatures can be parsed from the following elements in sig.xml:</p> <p><code>&lt;FrameSignature&gt;0  0  0  1  0  0  1  0  1  1  0  0  1  1 ...&lt;/FrameSignature&gt;</code></p> <p>Tip</p> <p>It is also possible to extract the signatures in a more compact binary format. But the format requires a custom binary parser to decode the frame signaturs.</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code","title":"<code>gen_video_code(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code_v0","title":"<code>gen_video_code_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.soft_hash_video_v0","title":"<code>soft_hash_video_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Compute video hash v0 from MP7 frame signatures.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>2D matrix of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code>"},{"location":"utilities/utils/","title":"ISCC - Utilities","text":""},{"location":"utilities/utils/#iscc_core.utils.json_canonical","title":"<code>json_canonical(obj)</code>","text":"<p>Canonical, deterministic serialization of ISCC metadata.</p> <p>We serialize ISCC metadata in a deterministic/reproducible manner by using JCS (RFC 8785) canonicalization.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def json_canonical(obj):\n# type: (Any) -&gt; bytes\n\"\"\"\n    Canonical, deterministic serialization of ISCC metadata.\n    We serialize ISCC metadata in a deterministic/reproducible manner by using\n    [JCS (RFC 8785)](https://datatracker.ietf.org/doc/html/rfc8785) canonicalization.\n    \"\"\"\nser = jcs.canonicalize(obj)\ndes = json.loads(ser)\nif des != obj:\nraise ValueError(f\"Not canonicalizable {obj} round-trips to {des}\")\nreturn ser\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.sliding_window","title":"<code>sliding_window(seq, width)</code>","text":"<p>Generate a sequence of equal \"width\" slices each advancing by one elemnt.</p> <p>All types that have a length and can be sliced are supported (list, tuple, str ...). The result type matches the type of the input sequence. Fragment slices smaller than the width at the end of the sequence are not produced. If \"witdh\" is smaller than the input sequence than one element will be returned that is shorter than the requested width.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>Sequence</code> <p>Sequence of values to slide over</p> required <code>width</code> <code>int</code> <p>Width of sliding window in number of items</p> required <p>Returns:</p> Type Description <code>Generator</code> <p>A generator of window sized items</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def sliding_window(seq, width):\n# type: (Sequence, int) -&gt; Generator\n\"\"\"\n    Generate a sequence of equal \"width\" slices each advancing by one elemnt.\n    All types that have a length and can be sliced are supported (list, tuple, str ...).\n    The result type matches the type of the input sequence.\n    Fragment slices smaller than the width at the end of the sequence are not produced.\n    If \"witdh\" is smaller than the input sequence than one element will be returned that\n    is shorter than the requested width.\n    :param Sequence seq: Sequence of values to slide over\n    :param int width: Width of sliding window in number of items\n    :returns: A generator of window sized items\n    :rtype: Generator\n    \"\"\"\nif width &lt; 2:\nraise AssertionError(\"Sliding window width must be 2 or bigger.\")\nidx = range(max(len(seq) - width + 1, 1))\nreturn (seq[i : i + width] for i in idx)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_compare","title":"<code>iscc_compare(a, b)</code>","text":"<p>Calculate separate hamming distances of compatible components of two ISCCs</p> <p>Returns:</p> Type Description <code>dict</code> <p>A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_compare(a, b):\n# type: (str, str) -&gt; dict\n\"\"\"\n    Calculate separate hamming distances of compatible components of two ISCCs\n    :return: A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match\n    :rtype: dict\n    \"\"\"\nac = [ic.Code(unit) for unit in ic.iscc_decompose(a)]\nbc = [ic.Code(unit) for unit in ic.iscc_decompose(b)]\nresult = {}\nfor ca in ac:\nfor cb in bc:\ncat = (ca.maintype, ca.subtype, ca.version)\ncbt = (cb.maintype, cb.subtype, ca.version)\nif cat == cbt:\nif ca.maintype != ic.MT.INSTANCE:\nresult[ca.maintype.name.lower() + \"_dist\"] = iscc_distance_bytes(\nca.hash_bytes, cb.hash_bytes\n)\nelse:\nresult[\"instance_match\"] = ca.hash_bytes == cb.hash_bytes\nreturn result\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_similarity","title":"<code>iscc_similarity(a, b)</code>","text":"<p>Calculate similarity of ISCC codes as a percentage value (0-100).</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Similarity of ISCC a and b in percent (based on hamming distance)</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_similarity(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate similarity of ISCC codes as a percentage value (0-100).\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Similarity of ISCC a and b in percent (based on hamming distance)\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nhdist = iscc_distance_bytes(a, b)\nnbits = len(a) * 8\nsim = int(((nbits - hdist) / nbits) * 100)\nreturn sim\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance","title":"<code>iscc_distance(a, b)</code>","text":"<p>Calculate hamming distance of ISCC codes.</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distanced in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate hamming distance of ISCC codes.\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Hamming distanced in number of bits.\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nreturn iscc_distance_bytes(a, b)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance_bytes","title":"<code>iscc_distance_bytes(a, b)</code>","text":"<p>Calculate hamming distance for binary hash digests of equal length.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <code>bytes</code> <p>binary hash digest</p> required <code>b</code> <code>bytes</code> <p>binary hash digest</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distance in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance_bytes(a, b):\n# type: (bytes, bytes) -&gt; int\n\"\"\"\n    Calculate hamming distance for binary hash digests of equal length.\n    :param bytes a: binary hash digest\n    :param bytes b: binary hash digest\n    :return: Hamming distance in number of bits.\n    :rtype: int\n    \"\"\"\nif len(a) != len(b):\nraise AssertionError(f\"Hash diggest of unequal length: {len(a)} vs {len(b)}\")\nba, bb = bitarray(), bitarray()\nba.frombytes(a)\nbb.frombytes(b)\nreturn count_xor(ba, bb)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_pair_unpack","title":"<code>iscc_pair_unpack(a, b)</code>","text":"<p>Unpack two ISCC codes and return their body hash digests if their headers match.</p> <p>Headers match if their MainType, SubType, and Version are identical.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>Tuple[bytes, bytes]</code> <p>Tuple with hash digests of a and b</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If ISCC headers don\u00b4t match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_pair_unpack(a, b):\n# type: (str, str) -&gt; Tuple[bytes, bytes]\n\"\"\"\n    Unpack two ISCC codes and return their body hash digests if their headers match.\n    Headers match if their MainType, SubType, and Version are identical.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Tuple with hash digests of a and b\n    :rtype: Tuple[bytes, bytes]\n    :raise ValueError: If ISCC headers don\u00b4t match\n    \"\"\"\na, b = ic.iscc_clean(ic.iscc_normalize(a)), ic.iscc_clean(ic.iscc_normalize(b))\na, b = ic.decode_base32(a), ic.decode_base32(b)\na, b = ic.decode_header(a), ic.decode_header(b)\nif not a[:-1] == b[:-1]:\nraise ValueError(f\"ISCC headers don\u00b4t match: {a}, {b}\")\nreturn a[-1], b[-1]\n</code></pre>"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 0925de6..73d6781 100755
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,137 +2,137 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://core.iscc.codes/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/changelog/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/conformance/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/constants/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/iscc_code/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/iscc_id/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/iso-reference/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/cdc/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/dct/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/minhash/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/simhash/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/wtahash/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/codec/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/includes/abbr/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/options/options/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_data/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_flake/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_instance/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_meta/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_audio/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_image/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_mixed/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_text/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_video/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/utilities/utils/</loc>
-         <lastmod>2023-12-07</lastmod>
+         <lastmod>2023-12-15</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 949ba4933aab9b30ae30e503cf58d0cc43914e6e..8d07a4a1a84f7448f64bed88d1ea829ee621f4f0 100755
GIT binary patch
delta 361
zcmV-v0ha#y0{a37ABzYGR!Mx32OWP&s!DAIWa|^ObL7|uoa&#{K0^BTH3YR;yHp1Q
zKAY3=$u`2(;c88Gfe_KVI^UJsJOih^M%UH(``c^rlpmT+H3SoZvXty%o%clN7a_)R
zEF%(Fd5)zIortPWohX(;u`QqTW|O6BDsYSDt1@0SQ<a#?C?BB2s8pHW3sHYmVtRU?
zEF)R3oBekGQ0(@_?lGk-V|Qt)m#%|0z-OZ+ny*USjocLChw&HCt&H&<Lz;K34^}z_
zEOC*fBZf;lg~usmN<+;N=8FI=1Yo(_r0IM>>TTM1TCw(NMXY@mT`yzL+8EKYH;yEE
z8mBwMPXwJZ!#szlQ{cz*XG&S8q~DeuaBxU$SyOz}k|b+jYvc#87UF!UQoNBCIIpuK
zaKxQA&*RcL%0Z)Fl^Ct;U{zv^SJ0~x6I|D|jW4Ky<2EKVNVr7*@CTin2-JK7t8VgG
HfDr%yU^cE1

delta 360
zcmV-u0hj*!0{Q|6ABzYG3e0hl2OWPT9r`0ewmv~SM~;2Kss2grBcyL%Lr|NwOLZ{d
zvpF4~Y$IGBuGVB12ob&8=i9Q%GjPglblpCGe|s&S@<X$!hhQR5mXclE=RMK+MTl`6
z%ZLP4o@41lC!*?8CyHfIRONHtY_fDs1#Z!N)yAu4suEKf<pY!$l`7MFA&P%mOi%BV
zWhCo$v#WLw#dcRzk11styGv8ObRDz-J{v93eAVJ^<fafmjK6?xWsL6_(!6VZu+k}D
ziHjs1F<jCqJWe4~8fuO(Uj%3&0L$GbP3HqrZ_~!pinUKGV(qi&dKr7x#)y`^aU{vp
zINce3BIuME<~cl_0zaNVQ#x2B{kH6YgF|A=n&P9DBv}JnBR_z(5a&ac;*GSxd7T}B
zBksI;9+%Eh4jTQc#Asy)s}fthf?k!F;JU7Dd_fHyw=tnX!X^5LKj_><pynGAl4zcQ
G5dZ*w9Hy@T