From d65414f69b66c687beb83a1976e0f3d5d3967cd7 Mon Sep 17 00:00:00 2001
From: Titusz Pan <titusz.pan@gmail.com>
Date: Sun, 17 Mar 2024 22:13:56 +0100
Subject: [PATCH] Deployed c5bebce with MkDocs version: 1.5.3

---
 changelog/index.html     |  13 +
 codec/index.html         | 618 ++++++++++++++++++++-------------------
 search/search_index.json |   2 +-
 sitemap.xml              |  54 ++--
 sitemap.xml.gz           | Bin 378 -> 379 bytes
 5 files changed, 351 insertions(+), 336 deletions(-)

diff --git a/changelog/index.html b/changelog/index.html
index ae355e8..0c266b5 100755
--- a/changelog/index.html
+++ b/changelog/index.html
@@ -1114,6 +1114,13 @@
     </label>
     <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
       
+        <li class="md-nav__item">
+  <a href="#109-2024-03-17" class="md-nav__link">
+    [1.0.9] - 2024-03-17
+  </a>
+  
+</li>
+      
         <li class="md-nav__item">
   <a href="#108-2024-01-30" class="md-nav__link">
     [1.0.8] - 2024-01-30
@@ -1372,6 +1379,12 @@
 
 
 <h1 id="changelog">Changelog<a class="headerlink" href="#changelog" title="Permanent link">#</a></h1>
+<h2 id="109-2024-03-17">[1.0.9] - 2024-03-17<a class="headerlink" href="#109-2024-03-17" title="Permanent link">#</a></h2>
+<ul>
+<li>Updated dependencies</li>
+<li>Fixed issue with decoding lower case base32hex strings</li>
+<li>Added support for MSCDI validation</li>
+</ul>
 <h2 id="108-2024-01-30">[1.0.8] - 2024-01-30<a class="headerlink" href="#108-2024-01-30" title="Permanent link">#</a></h2>
 <ul>
 <li>Added implementors guide to README.md</li>
diff --git a/codec/index.html b/codec/index.html
index d116546..6befd20 100755
--- a/codec/index.html
+++ b/codec/index.html
@@ -1469,7 +1469,8 @@ <h3 id="iscc_core.codec.encode_component" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-19">19</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-18">18</a></span>
+<span class="normal"><a href="#__codelineno-0-19">19</a></span>
 <span class="normal"><a href="#__codelineno-0-20">20</a></span>
 <span class="normal"><a href="#__codelineno-0-21">21</a></span>
 <span class="normal"><a href="#__codelineno-0-22">22</a></span>
@@ -1497,37 +1498,36 @@ <h3 id="iscc_core.codec.encode_component" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-44">44</a></span>
 <span class="normal"><a href="#__codelineno-0-45">45</a></span>
 <span class="normal"><a href="#__codelineno-0-46">46</a></span>
-<span class="normal"><a href="#__codelineno-0-47">47</a></span>
-<span class="normal"><a href="#__codelineno-0-48">48</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="k">def</span> <span class="nf">encode_component</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">bit_length</span><span class="p">,</span> <span class="n">digest</span><span class="p">):</span>
-<a id="__codelineno-0-20" name="__codelineno-0-20"></a>    <span class="c1"># type: (MainType, SubType, Version, Length, bytes) -&gt; str</span>
-<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd">    Encode an ISCC-UNIT inlcuding header and body with standard base32 encoding.</span>
-<a id="__codelineno-0-23" name="__codelineno-0-23"></a>
-<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd">    !!! note</span>
-<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd">        The `length` value must be the **length in number of bits** for the component.</span>
-<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd">        If `digest` has more bits than specified by `length` it wil be truncated.</span>
+<span class="normal"><a href="#__codelineno-0-47">47</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="k">def</span> <span class="nf">encode_component</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">bit_length</span><span class="p">,</span> <span class="n">digest</span><span class="p">):</span>
+<a id="__codelineno-0-19" name="__codelineno-0-19"></a>    <span class="c1"># type: (MainType, SubType, Version, Length, bytes) -&gt; str</span>
+<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd">    Encode an ISCC-UNIT inlcuding header and body with standard base32 encoding.</span>
+<a id="__codelineno-0-22" name="__codelineno-0-22"></a>
+<a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="sd">    !!! note</span>
+<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd">        The `length` value must be the **length in number of bits** for the component.</span>
+<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd">        If `digest` has more bits than specified by `length` it wil be truncated.</span>
+<a id="__codelineno-0-26" name="__codelineno-0-26"></a>
 <a id="__codelineno-0-27" name="__codelineno-0-27"></a>
-<a id="__codelineno-0-28" name="__codelineno-0-28"></a>
-<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd">    :param MainType mtype: Maintype of unit (0-6)</span>
-<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd">    :param SubType stype: SubType of unit depending on MainType (0-5)</span>
-<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd">    :param Version version: Version of unit algorithm (0).</span>
-<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd">    :param length bit_length: Length of unit, in number of bits (multiple of 32)</span>
-<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd">    :param bytes digest: The hash digest of the unit.</span>
-<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd">    :return: Base32 encoded ISCC-UNIT.</span>
-<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd">    :rtype: str</span>
-<a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-37" name="__codelineno-0-37"></a>    <span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">(</span><span class="n">MT</span><span class="o">.</span><span class="n">META</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">SEMANTIC</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">CONTENT</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">DATA</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">INSTANCE</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">ID</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">FLAKE</span><span class="p">):</span>
-<a id="__codelineno-0-38" name="__codelineno-0-38"></a>        <span class="n">encoded_length</span> <span class="o">=</span> <span class="n">encode_length</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">bit_length</span><span class="p">)</span>
-<a id="__codelineno-0-39" name="__codelineno-0-39"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ISCC</span><span class="p">:</span>
-<a id="__codelineno-0-40" name="__codelineno-0-40"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2"> is not a unit&quot;</span><span class="p">)</span>
-<a id="__codelineno-0-41" name="__codelineno-0-41"></a>    <span class="k">else</span><span class="p">:</span>
-<a id="__codelineno-0-42" name="__codelineno-0-42"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Illegal MainType </span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
-<a id="__codelineno-0-43" name="__codelineno-0-43"></a>
-<a id="__codelineno-0-44" name="__codelineno-0-44"></a>    <span class="n">nbytes</span> <span class="o">=</span> <span class="n">bit_length</span> <span class="o">//</span> <span class="mi">8</span>
-<a id="__codelineno-0-45" name="__codelineno-0-45"></a>    <span class="n">header</span> <span class="o">=</span> <span class="n">encode_header</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">encoded_length</span><span class="p">)</span>
-<a id="__codelineno-0-46" name="__codelineno-0-46"></a>    <span class="n">body</span> <span class="o">=</span> <span class="n">digest</span><span class="p">[:</span><span class="n">nbytes</span><span class="p">]</span>
-<a id="__codelineno-0-47" name="__codelineno-0-47"></a>    <span class="n">component_code</span> <span class="o">=</span> <span class="n">encode_base32</span><span class="p">(</span><span class="n">header</span> <span class="o">+</span> <span class="n">body</span><span class="p">)</span>
-<a id="__codelineno-0-48" name="__codelineno-0-48"></a>    <span class="k">return</span> <span class="n">component_code</span>
+<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd">    :param MainType mtype: Maintype of unit (0-6)</span>
+<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd">    :param SubType stype: SubType of unit depending on MainType (0-5)</span>
+<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd">    :param Version version: Version of unit algorithm (0).</span>
+<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd">    :param length bit_length: Length of unit, in number of bits (multiple of 32)</span>
+<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd">    :param bytes digest: The hash digest of the unit.</span>
+<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd">    :return: Base32 encoded ISCC-UNIT.</span>
+<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd">    :rtype: str</span>
+<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-36" name="__codelineno-0-36"></a>    <span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">(</span><span class="n">MT</span><span class="o">.</span><span class="n">META</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">SEMANTIC</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">CONTENT</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">DATA</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">INSTANCE</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">ID</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">FLAKE</span><span class="p">):</span>
+<a id="__codelineno-0-37" name="__codelineno-0-37"></a>        <span class="n">encoded_length</span> <span class="o">=</span> <span class="n">encode_length</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">bit_length</span><span class="p">)</span>
+<a id="__codelineno-0-38" name="__codelineno-0-38"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ISCC</span><span class="p">:</span>
+<a id="__codelineno-0-39" name="__codelineno-0-39"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2"> is not a unit&quot;</span><span class="p">)</span>
+<a id="__codelineno-0-40" name="__codelineno-0-40"></a>    <span class="k">else</span><span class="p">:</span>
+<a id="__codelineno-0-41" name="__codelineno-0-41"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Illegal MainType </span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+<a id="__codelineno-0-42" name="__codelineno-0-42"></a>
+<a id="__codelineno-0-43" name="__codelineno-0-43"></a>    <span class="n">nbytes</span> <span class="o">=</span> <span class="n">bit_length</span> <span class="o">//</span> <span class="mi">8</span>
+<a id="__codelineno-0-44" name="__codelineno-0-44"></a>    <span class="n">header</span> <span class="o">=</span> <span class="n">encode_header</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">encoded_length</span><span class="p">)</span>
+<a id="__codelineno-0-45" name="__codelineno-0-45"></a>    <span class="n">body</span> <span class="o">=</span> <span class="n">digest</span><span class="p">[:</span><span class="n">nbytes</span><span class="p">]</span>
+<a id="__codelineno-0-46" name="__codelineno-0-46"></a>    <span class="n">component_code</span> <span class="o">=</span> <span class="n">encode_base32</span><span class="p">(</span><span class="n">header</span> <span class="o">+</span> <span class="n">body</span><span class="p">)</span>
+<a id="__codelineno-0-47" name="__codelineno-0-47"></a>    <span class="k">return</span> <span class="n">component_code</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -1634,7 +1634,8 @@ <h3 id="iscc_core.codec.encode_header" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-51">51</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-50">50</a></span>
+<span class="normal"><a href="#__codelineno-0-51">51</a></span>
 <span class="normal"><a href="#__codelineno-0-52">52</a></span>
 <span class="normal"><a href="#__codelineno-0-53">53</a></span>
 <span class="normal"><a href="#__codelineno-0-54">54</a></span>
@@ -1657,32 +1658,31 @@ <h3 id="iscc_core.codec.encode_header" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-71">71</a></span>
 <span class="normal"><a href="#__codelineno-0-72">72</a></span>
 <span class="normal"><a href="#__codelineno-0-73">73</a></span>
-<span class="normal"><a href="#__codelineno-0-74">74</a></span>
-<span class="normal"><a href="#__codelineno-0-75">75</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="k">def</span> <span class="nf">encode_header</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
-<a id="__codelineno-0-52" name="__codelineno-0-52"></a>    <span class="c1"># type: (MainType, SubType, Version, Length) -&gt; bytes</span>
-<a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd">    Encodes header values with nibble-sized (4-bit) variable-length encoding.</span>
-<a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="sd">    The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles</span>
-<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd">    is uneven it is padded with 4-bit `0000` at the end.</span>
-<a id="__codelineno-0-57" name="__codelineno-0-57"></a>
-<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd">    !!! warning</span>
-<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd">        The length value must be encoded beforhand because its semantics depend on</span>
-<a id="__codelineno-0-60" name="__codelineno-0-60"></a><span class="sd">        the MainType (see `encode_length` function).</span>
-<a id="__codelineno-0-61" name="__codelineno-0-61"></a>
-<a id="__codelineno-0-62" name="__codelineno-0-62"></a><span class="sd">    :param MainType mtype: MainType of unit.</span>
-<a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="sd">    :param SubType stype: SubType of unit.</span>
-<a id="__codelineno-0-64" name="__codelineno-0-64"></a><span class="sd">    :param Version version: Version of component algorithm.</span>
-<a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="sd">    :param Length length: length value of unit (1 means 64-bits for standard units)</span>
-<a id="__codelineno-0-66" name="__codelineno-0-66"></a><span class="sd">    :return: Varnibble stream encoded ISCC header as bytes.</span>
-<a id="__codelineno-0-67" name="__codelineno-0-67"></a><span class="sd">    :rtype: bytes</span>
-<a id="__codelineno-0-68" name="__codelineno-0-68"></a>
-<a id="__codelineno-0-69" name="__codelineno-0-69"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-70" name="__codelineno-0-70"></a>    <span class="n">header</span> <span class="o">=</span> <span class="n">bitarray</span><span class="p">()</span>
-<a id="__codelineno-0-71" name="__codelineno-0-71"></a>    <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">length</span><span class="p">):</span>
-<a id="__codelineno-0-72" name="__codelineno-0-72"></a>        <span class="n">header</span> <span class="o">+=</span> <span class="n">encode_varnibble</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
-<a id="__codelineno-0-73" name="__codelineno-0-73"></a>    <span class="c1"># Append zero-padding if required (right side, least-significant bits).</span>
-<a id="__codelineno-0-74" name="__codelineno-0-74"></a>    <span class="n">header</span><span class="o">.</span><span class="n">fill</span><span class="p">()</span>
-<a id="__codelineno-0-75" name="__codelineno-0-75"></a>    <span class="k">return</span> <span class="n">header</span><span class="o">.</span><span class="n">tobytes</span><span class="p">()</span>
+<span class="normal"><a href="#__codelineno-0-74">74</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="k">def</span> <span class="nf">encode_header</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
+<a id="__codelineno-0-51" name="__codelineno-0-51"></a>    <span class="c1"># type: (MainType, SubType, Version, Length) -&gt; bytes</span>
+<a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="sd">    Encodes header values with nibble-sized (4-bit) variable-length encoding.</span>
+<a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd">    The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles</span>
+<a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="sd">    is uneven it is padded with 4-bit `0000` at the end.</span>
+<a id="__codelineno-0-56" name="__codelineno-0-56"></a>
+<a id="__codelineno-0-57" name="__codelineno-0-57"></a><span class="sd">    !!! warning</span>
+<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd">        The length value must be encoded beforhand because its semantics depend on</span>
+<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd">        the MainType (see `encode_length` function).</span>
+<a id="__codelineno-0-60" name="__codelineno-0-60"></a>
+<a id="__codelineno-0-61" name="__codelineno-0-61"></a><span class="sd">    :param MainType mtype: MainType of unit.</span>
+<a id="__codelineno-0-62" name="__codelineno-0-62"></a><span class="sd">    :param SubType stype: SubType of unit.</span>
+<a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="sd">    :param Version version: Version of component algorithm.</span>
+<a id="__codelineno-0-64" name="__codelineno-0-64"></a><span class="sd">    :param Length length: length value of unit (1 means 64-bits for standard units)</span>
+<a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="sd">    :return: Varnibble stream encoded ISCC header as bytes.</span>
+<a id="__codelineno-0-66" name="__codelineno-0-66"></a><span class="sd">    :rtype: bytes</span>
+<a id="__codelineno-0-67" name="__codelineno-0-67"></a>
+<a id="__codelineno-0-68" name="__codelineno-0-68"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-69" name="__codelineno-0-69"></a>    <span class="n">header</span> <span class="o">=</span> <span class="n">bitarray</span><span class="p">()</span>
+<a id="__codelineno-0-70" name="__codelineno-0-70"></a>    <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">stype</span><span class="p">,</span> <span class="n">version</span><span class="p">,</span> <span class="n">length</span><span class="p">):</span>
+<a id="__codelineno-0-71" name="__codelineno-0-71"></a>        <span class="n">header</span> <span class="o">+=</span> <span class="n">encode_varnibble</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
+<a id="__codelineno-0-72" name="__codelineno-0-72"></a>    <span class="c1"># Append zero-padding if required (right side, least-significant bits).</span>
+<a id="__codelineno-0-73" name="__codelineno-0-73"></a>    <span class="n">header</span><span class="o">.</span><span class="n">fill</span><span class="p">()</span>
+<a id="__codelineno-0-74" name="__codelineno-0-74"></a>    <span class="k">return</span> <span class="n">header</span><span class="o">.</span><span class="n">tobytes</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -1755,7 +1755,8 @@ <h3 id="iscc_core.codec.decode_header" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-109">109</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-108">108</a></span>
+<span class="normal"><a href="#__codelineno-0-109">109</a></span>
 <span class="normal"><a href="#__codelineno-0-110">110</a></span>
 <span class="normal"><a href="#__codelineno-0-111">111</a></span>
 <span class="normal"><a href="#__codelineno-0-112">112</a></span>
@@ -1781,35 +1782,34 @@ <h3 id="iscc_core.codec.decode_header" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-132">132</a></span>
 <span class="normal"><a href="#__codelineno-0-133">133</a></span>
 <span class="normal"><a href="#__codelineno-0-134">134</a></span>
-<span class="normal"><a href="#__codelineno-0-135">135</a></span>
-<span class="normal"><a href="#__codelineno-0-136">136</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="k">def</span> <span class="nf">decode_header</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
-<a id="__codelineno-0-110" name="__codelineno-0-110"></a>    <span class="c1"># type: (bytes) -&gt; IsccTuple</span>
-<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd">    Decodes varnibble encoded header and returns it together with `tail data`.</span>
-<a id="__codelineno-0-113" name="__codelineno-0-113"></a>
-<a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd">    Tail data is included to enable decoding of sequential ISCCs. The returned tail</span>
-<a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd">    data must be truncated to decode_length(r[0], r[3]) bits to recover the actual</span>
-<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd">    hash-bytes.</span>
-<a id="__codelineno-0-117" name="__codelineno-0-117"></a>
-<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd">    :param bytes data: ISCC bytes</span>
-<a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd">    :return: (MainType, SubType, Version, length, TailData)</span>
-<a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd">    :rtype: IsccTuple</span>
-<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-122" name="__codelineno-0-122"></a>    <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
-<a id="__codelineno-0-123" name="__codelineno-0-123"></a>    <span class="n">ba</span> <span class="o">=</span> <span class="n">bitarray</span><span class="p">()</span>
-<a id="__codelineno-0-124" name="__codelineno-0-124"></a>    <span class="n">ba</span><span class="o">.</span><span class="n">frombytes</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
-<a id="__codelineno-0-125" name="__codelineno-0-125"></a>    <span class="n">data</span> <span class="o">=</span> <span class="n">ba</span>
-<a id="__codelineno-0-126" name="__codelineno-0-126"></a>    <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">4</span><span class="p">):</span>
-<a id="__codelineno-0-127" name="__codelineno-0-127"></a>        <span class="n">value</span><span class="p">,</span> <span class="n">data</span> <span class="o">=</span> <span class="n">decode_varnibble</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
-<a id="__codelineno-0-128" name="__codelineno-0-128"></a>        <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
-<a id="__codelineno-0-129" name="__codelineno-0-129"></a>
-<a id="__codelineno-0-130" name="__codelineno-0-130"></a>    <span class="c1"># Strip 4-bit padding if required</span>
-<a id="__codelineno-0-131" name="__codelineno-0-131"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">%</span> <span class="mi">8</span> <span class="ow">and</span> <span class="n">data</span><span class="p">[:</span><span class="mi">4</span><span class="p">]</span> <span class="o">==</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;0000&quot;</span><span class="p">):</span>
-<a id="__codelineno-0-132" name="__codelineno-0-132"></a>        <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span>
-<a id="__codelineno-0-133" name="__codelineno-0-133"></a>
-<a id="__codelineno-0-134" name="__codelineno-0-134"></a>    <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">tobytes</span><span class="p">())</span>
-<a id="__codelineno-0-135" name="__codelineno-0-135"></a>
-<a id="__codelineno-0-136" name="__codelineno-0-136"></a>    <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-135">135</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="k">def</span> <span class="nf">decode_header</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
+<a id="__codelineno-0-109" name="__codelineno-0-109"></a>    <span class="c1"># type: (bytes) -&gt; IsccTuple</span>
+<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd">    Decodes varnibble encoded header and returns it together with `tail data`.</span>
+<a id="__codelineno-0-112" name="__codelineno-0-112"></a>
+<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd">    Tail data is included to enable decoding of sequential ISCCs. The returned tail</span>
+<a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd">    data must be truncated to decode_length(r[0], r[3]) bits to recover the actual</span>
+<a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd">    hash-bytes.</span>
+<a id="__codelineno-0-116" name="__codelineno-0-116"></a>
+<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd">    :param bytes data: ISCC bytes</span>
+<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd">    :return: (MainType, SubType, Version, length, TailData)</span>
+<a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd">    :rtype: IsccTuple</span>
+<a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-121" name="__codelineno-0-121"></a>    <span class="n">result</span> <span class="o">=</span> <span class="p">[]</span>
+<a id="__codelineno-0-122" name="__codelineno-0-122"></a>    <span class="n">ba</span> <span class="o">=</span> <span class="n">bitarray</span><span class="p">()</span>
+<a id="__codelineno-0-123" name="__codelineno-0-123"></a>    <span class="n">ba</span><span class="o">.</span><span class="n">frombytes</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+<a id="__codelineno-0-124" name="__codelineno-0-124"></a>    <span class="n">data</span> <span class="o">=</span> <span class="n">ba</span>
+<a id="__codelineno-0-125" name="__codelineno-0-125"></a>    <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">4</span><span class="p">):</span>
+<a id="__codelineno-0-126" name="__codelineno-0-126"></a>        <span class="n">value</span><span class="p">,</span> <span class="n">data</span> <span class="o">=</span> <span class="n">decode_varnibble</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+<a id="__codelineno-0-127" name="__codelineno-0-127"></a>        <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+<a id="__codelineno-0-128" name="__codelineno-0-128"></a>
+<a id="__codelineno-0-129" name="__codelineno-0-129"></a>    <span class="c1"># Strip 4-bit padding if required</span>
+<a id="__codelineno-0-130" name="__codelineno-0-130"></a>    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">%</span> <span class="mi">8</span> <span class="ow">and</span> <span class="n">data</span><span class="p">[:</span><span class="mi">4</span><span class="p">]</span> <span class="o">==</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;0000&quot;</span><span class="p">):</span>
+<a id="__codelineno-0-131" name="__codelineno-0-131"></a>        <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span>
+<a id="__codelineno-0-132" name="__codelineno-0-132"></a>
+<a id="__codelineno-0-133" name="__codelineno-0-133"></a>    <span class="n">result</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">tobytes</span><span class="p">())</span>
+<a id="__codelineno-0-134" name="__codelineno-0-134"></a>
+<a id="__codelineno-0-135" name="__codelineno-0-135"></a>    <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -1917,7 +1917,8 @@ <h3 id="iscc_core.codec.encode_varnibble" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
+<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
 <span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
 <span class="normal"><a href="#__codelineno-0-80"> 80</a></span>
 <span class="normal"><a href="#__codelineno-0-81"> 81</a></span>
@@ -1944,36 +1945,35 @@ <h3 id="iscc_core.codec.encode_varnibble" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-102">102</a></span>
 <span class="normal"><a href="#__codelineno-0-103">103</a></span>
 <span class="normal"><a href="#__codelineno-0-104">104</a></span>
-<span class="normal"><a href="#__codelineno-0-105">105</a></span>
-<span class="normal"><a href="#__codelineno-0-106">106</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-78" name="__codelineno-0-78"></a><span class="k">def</span> <span class="nf">encode_varnibble</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
-<a id="__codelineno-0-79" name="__codelineno-0-79"></a>    <span class="c1"># type: (int) -&gt; bitarray</span>
-<a id="__codelineno-0-80" name="__codelineno-0-80"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-81" name="__codelineno-0-81"></a><span class="sd">    Writes integer to variable length sequence of 4-bit chunks.</span>
-<a id="__codelineno-0-82" name="__codelineno-0-82"></a>
-<a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="sd">    Variable-length encoding scheme:</span>
-<a id="__codelineno-0-84" name="__codelineno-0-84"></a>
-<a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="sd">    ------------------------------------------------------</span>
-<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd">    | prefix bits | nibbles | data bits | unsigned range |</span>
-<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd">    | ----------- | ------- | --------- | -------------- |</span>
-<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd">    | 0           | 1       | 3         | 0 - 7          |</span>
-<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd">    | 10          | 2       | 6         | 8 - 71         |</span>
-<a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="sd">    | 110         | 3       | 9         | 72 - 583       |</span>
-<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd">    | 1110        | 4       | 12        | 584 - 4679     |</span>
-<a id="__codelineno-0-92" name="__codelineno-0-92"></a>
-<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd">    :param int n: Positive integer to be encoded as varnibble (0-4679)</span>
-<a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd">    :return: Varnibble encoded integera</span>
-<a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd">    :rtype: bitarray</span>
-<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-97" name="__codelineno-0-97"></a>    <span class="k">if</span> <span class="mi">0</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">8</span><span class="p">:</span>
-<a id="__codelineno-0-98" name="__codelineno-0-98"></a>        <span class="k">return</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
-<a id="__codelineno-0-99" name="__codelineno-0-99"></a>    <span class="k">elif</span> <span class="mi">8</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">72</span><span class="p">:</span>
-<a id="__codelineno-0-100" name="__codelineno-0-100"></a>        <span class="k">return</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;10&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">8</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">6</span><span class="p">)</span>
-<a id="__codelineno-0-101" name="__codelineno-0-101"></a>    <span class="k">elif</span> <span class="mi">72</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">584</span><span class="p">:</span>
-<a id="__codelineno-0-102" name="__codelineno-0-102"></a>        <span class="k">return</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;110&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">72</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">9</span><span class="p">)</span>
-<a id="__codelineno-0-103" name="__codelineno-0-103"></a>    <span class="k">elif</span> <span class="mi">584</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">4680</span><span class="p">:</span>
-<a id="__codelineno-0-104" name="__codelineno-0-104"></a>        <span class="k">return</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;1110&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">584</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">12</span><span class="p">)</span>
-<a id="__codelineno-0-105" name="__codelineno-0-105"></a>    <span class="k">else</span><span class="p">:</span>
-<a id="__codelineno-0-106" name="__codelineno-0-106"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Value must be between 0 and 4679&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-105">105</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-77" name="__codelineno-0-77"></a><span class="k">def</span> <span class="nf">encode_varnibble</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
+<a id="__codelineno-0-78" name="__codelineno-0-78"></a>    <span class="c1"># type: (int) -&gt; bitarray</span>
+<a id="__codelineno-0-79" name="__codelineno-0-79"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-80" name="__codelineno-0-80"></a><span class="sd">    Writes integer to variable length sequence of 4-bit chunks.</span>
+<a id="__codelineno-0-81" name="__codelineno-0-81"></a>
+<a id="__codelineno-0-82" name="__codelineno-0-82"></a><span class="sd">    Variable-length encoding scheme:</span>
+<a id="__codelineno-0-83" name="__codelineno-0-83"></a>
+<a id="__codelineno-0-84" name="__codelineno-0-84"></a><span class="sd">    ------------------------------------------------------</span>
+<a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="sd">    | prefix bits | nibbles | data bits | unsigned range |</span>
+<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd">    | ----------- | ------- | --------- | -------------- |</span>
+<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd">    | 0           | 1       | 3         | 0 - 7          |</span>
+<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd">    | 10          | 2       | 6         | 8 - 71         |</span>
+<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd">    | 110         | 3       | 9         | 72 - 583       |</span>
+<a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="sd">    | 1110        | 4       | 12        | 584 - 4679     |</span>
+<a id="__codelineno-0-91" name="__codelineno-0-91"></a>
+<a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd">    :param int n: Positive integer to be encoded as varnibble (0-4679)</span>
+<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd">    :return: Varnibble encoded integera</span>
+<a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd">    :rtype: bitarray</span>
+<a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-96" name="__codelineno-0-96"></a>    <span class="k">if</span> <span class="mi">0</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">8</span><span class="p">:</span>
+<a id="__codelineno-0-97" name="__codelineno-0-97"></a>        <span class="k">return</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
+<a id="__codelineno-0-98" name="__codelineno-0-98"></a>    <span class="k">elif</span> <span class="mi">8</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">72</span><span class="p">:</span>
+<a id="__codelineno-0-99" name="__codelineno-0-99"></a>        <span class="k">return</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;10&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">8</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">6</span><span class="p">)</span>
+<a id="__codelineno-0-100" name="__codelineno-0-100"></a>    <span class="k">elif</span> <span class="mi">72</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">584</span><span class="p">:</span>
+<a id="__codelineno-0-101" name="__codelineno-0-101"></a>        <span class="k">return</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;110&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">72</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">9</span><span class="p">)</span>
+<a id="__codelineno-0-102" name="__codelineno-0-102"></a>    <span class="k">elif</span> <span class="mi">584</span> <span class="o">&lt;=</span> <span class="n">n</span> <span class="o">&lt;</span> <span class="mi">4680</span><span class="p">:</span>
+<a id="__codelineno-0-103" name="__codelineno-0-103"></a>        <span class="k">return</span> <span class="n">bitarray</span><span class="p">(</span><span class="s2">&quot;1110&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="n">int2ba</span><span class="p">(</span><span class="n">n</span> <span class="o">-</span> <span class="mi">584</span><span class="p">,</span> <span class="n">length</span><span class="o">=</span><span class="mi">12</span><span class="p">)</span>
+<a id="__codelineno-0-104" name="__codelineno-0-104"></a>    <span class="k">else</span><span class="p">:</span>
+<a id="__codelineno-0-105" name="__codelineno-0-105"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Value must be between 0 and 4679&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2043,7 +2043,8 @@ <h3 id="iscc_core.codec.decode_varnibble" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-139">139</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-138">138</a></span>
+<span class="normal"><a href="#__codelineno-0-139">139</a></span>
 <span class="normal"><a href="#__codelineno-0-140">140</a></span>
 <span class="normal"><a href="#__codelineno-0-141">141</a></span>
 <span class="normal"><a href="#__codelineno-0-142">142</a></span>
@@ -2062,28 +2063,27 @@ <h3 id="iscc_core.codec.decode_varnibble" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-155">155</a></span>
 <span class="normal"><a href="#__codelineno-0-156">156</a></span>
 <span class="normal"><a href="#__codelineno-0-157">157</a></span>
-<span class="normal"><a href="#__codelineno-0-158">158</a></span>
-<span class="normal"><a href="#__codelineno-0-159">159</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-139" name="__codelineno-0-139"></a><span class="k">def</span> <span class="nf">decode_varnibble</span><span class="p">(</span><span class="n">b</span><span class="p">):</span>
-<a id="__codelineno-0-140" name="__codelineno-0-140"></a>    <span class="c1"># type: (bitarray) -&gt; Tuple[int, bitarray]</span>
-<a id="__codelineno-0-141" name="__codelineno-0-141"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Reads first varnibble, returns its integer value and remaining bits.</span>
-<a id="__codelineno-0-142" name="__codelineno-0-142"></a>
-<a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="sd">    :param bitarray b: Array of header bits</span>
-<a id="__codelineno-0-144" name="__codelineno-0-144"></a><span class="sd">    :return: A tuple of the integer value of first varnible and the remaining bits.</span>
-<a id="__codelineno-0-145" name="__codelineno-0-145"></a><span class="sd">    :rtype: Tuple[int, bitarray]</span>
-<a id="__codelineno-0-146" name="__codelineno-0-146"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-147" name="__codelineno-0-147"></a>
-<a id="__codelineno-0-148" name="__codelineno-0-148"></a>    <span class="n">bits</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">b</span><span class="p">)</span>
-<a id="__codelineno-0-149" name="__codelineno-0-149"></a>
-<a id="__codelineno-0-150" name="__codelineno-0-150"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">4</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<a id="__codelineno-0-151" name="__codelineno-0-151"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[:</span><span class="mi">4</span><span class="p">]),</span> <span class="n">b</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span>
-<a id="__codelineno-0-152" name="__codelineno-0-152"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">8</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<a id="__codelineno-0-153" name="__codelineno-0-153"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="mi">2</span><span class="p">:</span><span class="mi">8</span><span class="p">])</span> <span class="o">+</span> <span class="mi">8</span><span class="p">,</span> <span class="n">b</span><span class="p">[</span><span class="mi">8</span><span class="p">:]</span>
-<a id="__codelineno-0-154" name="__codelineno-0-154"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">12</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<a id="__codelineno-0-155" name="__codelineno-0-155"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="mi">3</span><span class="p">:</span><span class="mi">12</span><span class="p">])</span> <span class="o">+</span> <span class="mi">72</span><span class="p">,</span> <span class="n">b</span><span class="p">[</span><span class="mi">12</span><span class="p">:]</span>
-<a id="__codelineno-0-156" name="__codelineno-0-156"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">16</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-<a id="__codelineno-0-157" name="__codelineno-0-157"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="mi">4</span><span class="p">:</span><span class="mi">16</span><span class="p">])</span> <span class="o">+</span> <span class="mi">584</span><span class="p">,</span> <span class="n">b</span><span class="p">[</span><span class="mi">16</span><span class="p">:]</span>
-<a id="__codelineno-0-158" name="__codelineno-0-158"></a>
-<a id="__codelineno-0-159" name="__codelineno-0-159"></a>    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Invalid bitarray&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-158">158</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-138" name="__codelineno-0-138"></a><span class="k">def</span> <span class="nf">decode_varnibble</span><span class="p">(</span><span class="n">b</span><span class="p">):</span>
+<a id="__codelineno-0-139" name="__codelineno-0-139"></a>    <span class="c1"># type: (bitarray) -&gt; Tuple[int, bitarray]</span>
+<a id="__codelineno-0-140" name="__codelineno-0-140"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;Reads first varnibble, returns its integer value and remaining bits.</span>
+<a id="__codelineno-0-141" name="__codelineno-0-141"></a>
+<a id="__codelineno-0-142" name="__codelineno-0-142"></a><span class="sd">    :param bitarray b: Array of header bits</span>
+<a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="sd">    :return: A tuple of the integer value of first varnible and the remaining bits.</span>
+<a id="__codelineno-0-144" name="__codelineno-0-144"></a><span class="sd">    :rtype: Tuple[int, bitarray]</span>
+<a id="__codelineno-0-145" name="__codelineno-0-145"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-146" name="__codelineno-0-146"></a>
+<a id="__codelineno-0-147" name="__codelineno-0-147"></a>    <span class="n">bits</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">b</span><span class="p">)</span>
+<a id="__codelineno-0-148" name="__codelineno-0-148"></a>
+<a id="__codelineno-0-149" name="__codelineno-0-149"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">4</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<a id="__codelineno-0-150" name="__codelineno-0-150"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[:</span><span class="mi">4</span><span class="p">]),</span> <span class="n">b</span><span class="p">[</span><span class="mi">4</span><span class="p">:]</span>
+<a id="__codelineno-0-151" name="__codelineno-0-151"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">8</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<a id="__codelineno-0-152" name="__codelineno-0-152"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="mi">2</span><span class="p">:</span><span class="mi">8</span><span class="p">])</span> <span class="o">+</span> <span class="mi">8</span><span class="p">,</span> <span class="n">b</span><span class="p">[</span><span class="mi">8</span><span class="p">:]</span>
+<a id="__codelineno-0-153" name="__codelineno-0-153"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">12</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<a id="__codelineno-0-154" name="__codelineno-0-154"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="mi">3</span><span class="p">:</span><span class="mi">12</span><span class="p">])</span> <span class="o">+</span> <span class="mi">72</span><span class="p">,</span> <span class="n">b</span><span class="p">[</span><span class="mi">12</span><span class="p">:]</span>
+<a id="__codelineno-0-155" name="__codelineno-0-155"></a>    <span class="k">if</span> <span class="n">bits</span> <span class="o">&gt;=</span> <span class="mi">16</span> <span class="ow">and</span> <span class="n">b</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+<a id="__codelineno-0-156" name="__codelineno-0-156"></a>        <span class="k">return</span> <span class="n">ba2int</span><span class="p">(</span><span class="n">b</span><span class="p">[</span><span class="mi">4</span><span class="p">:</span><span class="mi">16</span><span class="p">])</span> <span class="o">+</span> <span class="mi">584</span><span class="p">,</span> <span class="n">b</span><span class="p">[</span><span class="mi">16</span><span class="p">:]</span>
+<a id="__codelineno-0-157" name="__codelineno-0-157"></a>
+<a id="__codelineno-0-158" name="__codelineno-0-158"></a>    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Invalid bitarray&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2154,7 +2154,8 @@ <h3 id="iscc_core.codec.encode_units" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-162">162</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-161">161</a></span>
+<span class="normal"><a href="#__codelineno-0-162">162</a></span>
 <span class="normal"><a href="#__codelineno-0-163">163</a></span>
 <span class="normal"><a href="#__codelineno-0-164">164</a></span>
 <span class="normal"><a href="#__codelineno-0-165">165</a></span>
@@ -2163,18 +2164,17 @@ <h3 id="iscc_core.codec.encode_units" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-168">168</a></span>
 <span class="normal"><a href="#__codelineno-0-169">169</a></span>
 <span class="normal"><a href="#__codelineno-0-170">170</a></span>
-<span class="normal"><a href="#__codelineno-0-171">171</a></span>
-<span class="normal"><a href="#__codelineno-0-172">172</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-162" name="__codelineno-0-162"></a><span class="k">def</span> <span class="nf">encode_units</span><span class="p">(</span><span class="n">units</span><span class="p">):</span>
-<a id="__codelineno-0-163" name="__codelineno-0-163"></a>    <span class="c1"># type: (Tuple[MT, ...]) -&gt; int</span>
-<a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-165" name="__codelineno-0-165"></a><span class="sd">    Encodes a combination of ISCC units to an integer between 0-7 to be used as length</span>
-<a id="__codelineno-0-166" name="__codelineno-0-166"></a><span class="sd">    value for the final encoding of MT.ISCC</span>
-<a id="__codelineno-0-167" name="__codelineno-0-167"></a>
-<a id="__codelineno-0-168" name="__codelineno-0-168"></a><span class="sd">    :param Tuple units: A tuple of a MainType combination (can be empty)</span>
-<a id="__codelineno-0-169" name="__codelineno-0-169"></a><span class="sd">    :return: Integer value to be used as length-value for header encoding</span>
-<a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="sd">    :rtype: int</span>
-<a id="__codelineno-0-171" name="__codelineno-0-171"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-172" name="__codelineno-0-172"></a>    <span class="k">return</span> <span class="n">UNITS</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">units</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-171">171</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-161" name="__codelineno-0-161"></a><span class="k">def</span> <span class="nf">encode_units</span><span class="p">(</span><span class="n">units</span><span class="p">):</span>
+<a id="__codelineno-0-162" name="__codelineno-0-162"></a>    <span class="c1"># type: (Tuple[MT, ...]) -&gt; int</span>
+<a id="__codelineno-0-163" name="__codelineno-0-163"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-164" name="__codelineno-0-164"></a><span class="sd">    Encodes a combination of ISCC units to an integer between 0-7 to be used as length</span>
+<a id="__codelineno-0-165" name="__codelineno-0-165"></a><span class="sd">    value for the final encoding of MT.ISCC</span>
+<a id="__codelineno-0-166" name="__codelineno-0-166"></a>
+<a id="__codelineno-0-167" name="__codelineno-0-167"></a><span class="sd">    :param Tuple units: A tuple of a MainType combination (can be empty)</span>
+<a id="__codelineno-0-168" name="__codelineno-0-168"></a><span class="sd">    :return: Integer value to be used as length-value for header encoding</span>
+<a id="__codelineno-0-169" name="__codelineno-0-169"></a><span class="sd">    :rtype: int</span>
+<a id="__codelineno-0-170" name="__codelineno-0-170"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-171" name="__codelineno-0-171"></a>    <span class="k">return</span> <span class="n">UNITS</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">units</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2203,21 +2203,21 @@ <h3 id="iscc_core.codec.decode_units" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-175">175</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-174">174</a></span>
+<span class="normal"><a href="#__codelineno-0-175">175</a></span>
 <span class="normal"><a href="#__codelineno-0-176">176</a></span>
 <span class="normal"><a href="#__codelineno-0-177">177</a></span>
 <span class="normal"><a href="#__codelineno-0-178">178</a></span>
 <span class="normal"><a href="#__codelineno-0-179">179</a></span>
 <span class="normal"><a href="#__codelineno-0-180">180</a></span>
-<span class="normal"><a href="#__codelineno-0-181">181</a></span>
-<span class="normal"><a href="#__codelineno-0-182">182</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-175" name="__codelineno-0-175"></a><span class="k">def</span> <span class="nf">decode_units</span><span class="p">(</span><span class="n">unit_id</span><span class="p">):</span>
-<a id="__codelineno-0-176" name="__codelineno-0-176"></a>    <span class="c1"># type: (int) -&gt; Tuple[MT, ...]</span>
-<a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">    Decodes an ISCC header length value that has been encoded with a unit_id to an</span>
-<a id="__codelineno-0-179" name="__codelineno-0-179"></a><span class="sd">    ordered tuple of MainTypes.</span>
-<a id="__codelineno-0-180" name="__codelineno-0-180"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-181" name="__codelineno-0-181"></a>    <span class="n">units</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">UNITS</span><span class="p">[</span><span class="n">unit_id</span><span class="p">])</span>
-<a id="__codelineno-0-182" name="__codelineno-0-182"></a>    <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">MT</span><span class="p">(</span><span class="n">u</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span> <span class="ow">in</span> <span class="n">units</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-181">181</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-174" name="__codelineno-0-174"></a><span class="k">def</span> <span class="nf">decode_units</span><span class="p">(</span><span class="n">unit_id</span><span class="p">):</span>
+<a id="__codelineno-0-175" name="__codelineno-0-175"></a>    <span class="c1"># type: (int) -&gt; Tuple[MT, ...]</span>
+<a id="__codelineno-0-176" name="__codelineno-0-176"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-177" name="__codelineno-0-177"></a><span class="sd">    Decodes an ISCC header length value that has been encoded with a unit_id to an</span>
+<a id="__codelineno-0-178" name="__codelineno-0-178"></a><span class="sd">    ordered tuple of MainTypes.</span>
+<a id="__codelineno-0-179" name="__codelineno-0-179"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-180" name="__codelineno-0-180"></a>    <span class="n">units</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">UNITS</span><span class="p">[</span><span class="n">unit_id</span><span class="p">])</span>
+<a id="__codelineno-0-181" name="__codelineno-0-181"></a>    <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">MT</span><span class="p">(</span><span class="n">u</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span> <span class="ow">in</span> <span class="n">units</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2326,7 +2326,8 @@ <h3 id="iscc_core.codec.encode_length" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-185">185</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-184">184</a></span>
+<span class="normal"><a href="#__codelineno-0-185">185</a></span>
 <span class="normal"><a href="#__codelineno-0-186">186</a></span>
 <span class="normal"><a href="#__codelineno-0-187">187</a></span>
 <span class="normal"><a href="#__codelineno-0-188">188</a></span>
@@ -2386,69 +2387,68 @@ <h3 id="iscc_core.codec.encode_length" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-242">242</a></span>
 <span class="normal"><a href="#__codelineno-0-243">243</a></span>
 <span class="normal"><a href="#__codelineno-0-244">244</a></span>
-<span class="normal"><a href="#__codelineno-0-245">245</a></span>
-<span class="normal"><a href="#__codelineno-0-246">246</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-185" name="__codelineno-0-185"></a><span class="k">def</span> <span class="nf">encode_length</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">length</span><span class="p">):</span>
-<a id="__codelineno-0-186" name="__codelineno-0-186"></a>    <span class="c1"># type: (MainType, Length) -&gt; int</span>
-<a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="sd">    Encode length to integer value for header encoding.</span>
-<a id="__codelineno-0-189" name="__codelineno-0-189"></a>
-<a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="sd">    The `length` value has MainType-specific semantics:</span>
-<a id="__codelineno-0-191" name="__codelineno-0-191"></a>
-<a id="__codelineno-0-192" name="__codelineno-0-192"></a><span class="sd">    For MainTypes `META`, `SEMANTIC`, `CONTENT`, `DATA`, `INSTANCE`:</span>
-<a id="__codelineno-0-193" name="__codelineno-0-193"></a>
-<a id="__codelineno-0-194" name="__codelineno-0-194"></a><span class="sd">        Length means number of bits for the body.</span>
-<a id="__codelineno-0-195" name="__codelineno-0-195"></a><span class="sd">        Length is encoded as the multiple of 32-bit chunks (0 being 32bits)</span>
-<a id="__codelineno-0-196" name="__codelineno-0-196"></a><span class="sd">        Examples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...</span>
-<a id="__codelineno-0-197" name="__codelineno-0-197"></a>
-<a id="__codelineno-0-198" name="__codelineno-0-198"></a><span class="sd">    For MainType `ISCC`:</span>
-<a id="__codelineno-0-199" name="__codelineno-0-199"></a>
-<a id="__codelineno-0-200" name="__codelineno-0-200"></a><span class="sd">        MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are</span>
-<a id="__codelineno-0-201" name="__codelineno-0-201"></a><span class="sd">        optional. Length means the composition of optional 64-bit units included</span>
-<a id="__codelineno-0-202" name="__codelineno-0-202"></a><span class="sd">        in the ISCC composite.</span>
-<a id="__codelineno-0-203" name="__codelineno-0-203"></a>
-<a id="__codelineno-0-204" name="__codelineno-0-204"></a><span class="sd">        Examples:</span>
-<a id="__codelineno-0-205" name="__codelineno-0-205"></a><span class="sd">            No optional units      -&gt; 0000 -&gt; 0</span>
-<a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="sd">            CONTENT                -&gt; 0001 -&gt; 1</span>
-<a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="sd">            SEMANTIC               -&gt; 0010 -&gt; 2</span>
-<a id="__codelineno-0-208" name="__codelineno-0-208"></a><span class="sd">            SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3</span>
-<a id="__codelineno-0-209" name="__codelineno-0-209"></a><span class="sd">            META                   -&gt; 0100 -&gt; 4</span>
-<a id="__codelineno-0-210" name="__codelineno-0-210"></a><span class="sd">            META, CONTENT          -&gt; 0101 -&gt; 5</span>
-<a id="__codelineno-0-211" name="__codelineno-0-211"></a><span class="sd">            ...</span>
-<a id="__codelineno-0-212" name="__codelineno-0-212"></a>
-<a id="__codelineno-0-213" name="__codelineno-0-213"></a><span class="sd">    For MainType `ID`:</span>
-<a id="__codelineno-0-214" name="__codelineno-0-214"></a>
-<a id="__codelineno-0-215" name="__codelineno-0-215"></a><span class="sd">        Lengths means number the number of bits for the body including the counter</span>
-<a id="__codelineno-0-216" name="__codelineno-0-216"></a><span class="sd">        Length is encoded as number of bytes of the counter (64-bit body is implicit)</span>
-<a id="__codelineno-0-217" name="__codelineno-0-217"></a><span class="sd">        Examples:</span>
-<a id="__codelineno-0-218" name="__codelineno-0-218"></a><span class="sd">            64 -&gt; 0 (No counter)</span>
-<a id="__codelineno-0-219" name="__codelineno-0-219"></a><span class="sd">            72 -&gt; 1 (One byte counter)</span>
-<a id="__codelineno-0-220" name="__codelineno-0-220"></a><span class="sd">            80 -&gt; 2 (Two byte counter)</span>
-<a id="__codelineno-0-221" name="__codelineno-0-221"></a><span class="sd">            ...</span>
-<a id="__codelineno-0-222" name="__codelineno-0-222"></a>
-<a id="__codelineno-0-223" name="__codelineno-0-223"></a><span class="sd">    :param MainType mtype: The MainType for which to encode the length value.</span>
-<a id="__codelineno-0-224" name="__codelineno-0-224"></a><span class="sd">    :param Length length: The length expressed according to the semantics of the type</span>
-<a id="__codelineno-0-225" name="__codelineno-0-225"></a><span class="sd">    :return: The length value encoded as integer for use with write_header.</span>
-<a id="__codelineno-0-226" name="__codelineno-0-226"></a><span class="sd">    :rtype: int</span>
-<a id="__codelineno-0-227" name="__codelineno-0-227"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-228" name="__codelineno-0-228"></a>
-<a id="__codelineno-0-229" name="__codelineno-0-229"></a>    <span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Invalid length </span><span class="si">{</span><span class="n">length</span><span class="si">}</span><span class="s2"> for MainType </span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2">&quot;</span>
-<a id="__codelineno-0-230" name="__codelineno-0-230"></a>    <span class="c1"># standard case (length field denotes number of 32-bit chunks, 0 being 32-bits)</span>
-<a id="__codelineno-0-231" name="__codelineno-0-231"></a>    <span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">(</span><span class="n">MT</span><span class="o">.</span><span class="n">META</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">SEMANTIC</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">CONTENT</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">DATA</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">INSTANCE</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">FLAKE</span><span class="p">):</span>
-<a id="__codelineno-0-232" name="__codelineno-0-232"></a>        <span class="k">if</span> <span class="n">length</span> <span class="o">&gt;=</span> <span class="mi">32</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">length</span> <span class="o">%</span> <span class="mi">32</span><span class="p">:</span>
-<a id="__codelineno-0-233" name="__codelineno-0-233"></a>            <span class="k">return</span> <span class="p">(</span><span class="n">length</span> <span class="o">//</span> <span class="mi">32</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
-<a id="__codelineno-0-234" name="__codelineno-0-234"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
-<a id="__codelineno-0-235" name="__codelineno-0-235"></a>    <span class="c1"># flag type encoding of included components (pass through as encoded out-of-band)</span>
-<a id="__codelineno-0-236" name="__codelineno-0-236"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ISCC</span><span class="p">:</span>
-<a id="__codelineno-0-237" name="__codelineno-0-237"></a>        <span class="k">if</span> <span class="mi">0</span> <span class="o">&lt;=</span> <span class="n">length</span> <span class="o">&lt;=</span> <span class="mi">7</span><span class="p">:</span>
-<a id="__codelineno-0-238" name="__codelineno-0-238"></a>            <span class="k">return</span> <span class="n">length</span>
-<a id="__codelineno-0-239" name="__codelineno-0-239"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
-<a id="__codelineno-0-240" name="__codelineno-0-240"></a>    <span class="c1"># counter byte length encoding</span>
-<a id="__codelineno-0-241" name="__codelineno-0-241"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ID</span><span class="p">:</span>
-<a id="__codelineno-0-242" name="__codelineno-0-242"></a>        <span class="k">if</span> <span class="mi">64</span> <span class="o">&lt;=</span> <span class="n">length</span> <span class="o">&lt;=</span> <span class="mi">96</span><span class="p">:</span>
-<a id="__codelineno-0-243" name="__codelineno-0-243"></a>            <span class="k">return</span> <span class="p">(</span><span class="n">length</span> <span class="o">-</span> <span class="mi">64</span><span class="p">)</span> <span class="o">//</span> <span class="mi">8</span>
-<a id="__codelineno-0-244" name="__codelineno-0-244"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
-<a id="__codelineno-0-245" name="__codelineno-0-245"></a>    <span class="k">else</span><span class="p">:</span>
-<a id="__codelineno-0-246" name="__codelineno-0-246"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-245">245</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-184" name="__codelineno-0-184"></a><span class="k">def</span> <span class="nf">encode_length</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">length</span><span class="p">):</span>
+<a id="__codelineno-0-185" name="__codelineno-0-185"></a>    <span class="c1"># type: (MainType, Length) -&gt; int</span>
+<a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="sd">    Encode length to integer value for header encoding.</span>
+<a id="__codelineno-0-188" name="__codelineno-0-188"></a>
+<a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="sd">    The `length` value has MainType-specific semantics:</span>
+<a id="__codelineno-0-190" name="__codelineno-0-190"></a>
+<a id="__codelineno-0-191" name="__codelineno-0-191"></a><span class="sd">    For MainTypes `META`, `SEMANTIC`, `CONTENT`, `DATA`, `INSTANCE`:</span>
+<a id="__codelineno-0-192" name="__codelineno-0-192"></a>
+<a id="__codelineno-0-193" name="__codelineno-0-193"></a><span class="sd">        Length means number of bits for the body.</span>
+<a id="__codelineno-0-194" name="__codelineno-0-194"></a><span class="sd">        Length is encoded as the multiple of 32-bit chunks (0 being 32bits)</span>
+<a id="__codelineno-0-195" name="__codelineno-0-195"></a><span class="sd">        Examples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...</span>
+<a id="__codelineno-0-196" name="__codelineno-0-196"></a>
+<a id="__codelineno-0-197" name="__codelineno-0-197"></a><span class="sd">    For MainType `ISCC`:</span>
+<a id="__codelineno-0-198" name="__codelineno-0-198"></a>
+<a id="__codelineno-0-199" name="__codelineno-0-199"></a><span class="sd">        MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are</span>
+<a id="__codelineno-0-200" name="__codelineno-0-200"></a><span class="sd">        optional. Length means the composition of optional 64-bit units included</span>
+<a id="__codelineno-0-201" name="__codelineno-0-201"></a><span class="sd">        in the ISCC composite.</span>
+<a id="__codelineno-0-202" name="__codelineno-0-202"></a>
+<a id="__codelineno-0-203" name="__codelineno-0-203"></a><span class="sd">        Examples:</span>
+<a id="__codelineno-0-204" name="__codelineno-0-204"></a><span class="sd">            No optional units      -&gt; 0000 -&gt; 0</span>
+<a id="__codelineno-0-205" name="__codelineno-0-205"></a><span class="sd">            CONTENT                -&gt; 0001 -&gt; 1</span>
+<a id="__codelineno-0-206" name="__codelineno-0-206"></a><span class="sd">            SEMANTIC               -&gt; 0010 -&gt; 2</span>
+<a id="__codelineno-0-207" name="__codelineno-0-207"></a><span class="sd">            SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3</span>
+<a id="__codelineno-0-208" name="__codelineno-0-208"></a><span class="sd">            META                   -&gt; 0100 -&gt; 4</span>
+<a id="__codelineno-0-209" name="__codelineno-0-209"></a><span class="sd">            META, CONTENT          -&gt; 0101 -&gt; 5</span>
+<a id="__codelineno-0-210" name="__codelineno-0-210"></a><span class="sd">            ...</span>
+<a id="__codelineno-0-211" name="__codelineno-0-211"></a>
+<a id="__codelineno-0-212" name="__codelineno-0-212"></a><span class="sd">    For MainType `ID`:</span>
+<a id="__codelineno-0-213" name="__codelineno-0-213"></a>
+<a id="__codelineno-0-214" name="__codelineno-0-214"></a><span class="sd">        Lengths means number the number of bits for the body including the counter</span>
+<a id="__codelineno-0-215" name="__codelineno-0-215"></a><span class="sd">        Length is encoded as number of bytes of the counter (64-bit body is implicit)</span>
+<a id="__codelineno-0-216" name="__codelineno-0-216"></a><span class="sd">        Examples:</span>
+<a id="__codelineno-0-217" name="__codelineno-0-217"></a><span class="sd">            64 -&gt; 0 (No counter)</span>
+<a id="__codelineno-0-218" name="__codelineno-0-218"></a><span class="sd">            72 -&gt; 1 (One byte counter)</span>
+<a id="__codelineno-0-219" name="__codelineno-0-219"></a><span class="sd">            80 -&gt; 2 (Two byte counter)</span>
+<a id="__codelineno-0-220" name="__codelineno-0-220"></a><span class="sd">            ...</span>
+<a id="__codelineno-0-221" name="__codelineno-0-221"></a>
+<a id="__codelineno-0-222" name="__codelineno-0-222"></a><span class="sd">    :param MainType mtype: The MainType for which to encode the length value.</span>
+<a id="__codelineno-0-223" name="__codelineno-0-223"></a><span class="sd">    :param Length length: The length expressed according to the semantics of the type</span>
+<a id="__codelineno-0-224" name="__codelineno-0-224"></a><span class="sd">    :return: The length value encoded as integer for use with write_header.</span>
+<a id="__codelineno-0-225" name="__codelineno-0-225"></a><span class="sd">    :rtype: int</span>
+<a id="__codelineno-0-226" name="__codelineno-0-226"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-227" name="__codelineno-0-227"></a>
+<a id="__codelineno-0-228" name="__codelineno-0-228"></a>    <span class="n">error</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Invalid length </span><span class="si">{</span><span class="n">length</span><span class="si">}</span><span class="s2"> for MainType </span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2">&quot;</span>
+<a id="__codelineno-0-229" name="__codelineno-0-229"></a>    <span class="c1"># standard case (length field denotes number of 32-bit chunks, 0 being 32-bits)</span>
+<a id="__codelineno-0-230" name="__codelineno-0-230"></a>    <span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">(</span><span class="n">MT</span><span class="o">.</span><span class="n">META</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">SEMANTIC</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">CONTENT</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">DATA</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">INSTANCE</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">FLAKE</span><span class="p">):</span>
+<a id="__codelineno-0-231" name="__codelineno-0-231"></a>        <span class="k">if</span> <span class="n">length</span> <span class="o">&gt;=</span> <span class="mi">32</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">length</span> <span class="o">%</span> <span class="mi">32</span><span class="p">:</span>
+<a id="__codelineno-0-232" name="__codelineno-0-232"></a>            <span class="k">return</span> <span class="p">(</span><span class="n">length</span> <span class="o">//</span> <span class="mi">32</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
+<a id="__codelineno-0-233" name="__codelineno-0-233"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
+<a id="__codelineno-0-234" name="__codelineno-0-234"></a>    <span class="c1"># flag type encoding of included components (pass through as encoded out-of-band)</span>
+<a id="__codelineno-0-235" name="__codelineno-0-235"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ISCC</span><span class="p">:</span>
+<a id="__codelineno-0-236" name="__codelineno-0-236"></a>        <span class="k">if</span> <span class="mi">0</span> <span class="o">&lt;=</span> <span class="n">length</span> <span class="o">&lt;=</span> <span class="mi">7</span><span class="p">:</span>
+<a id="__codelineno-0-237" name="__codelineno-0-237"></a>            <span class="k">return</span> <span class="n">length</span>
+<a id="__codelineno-0-238" name="__codelineno-0-238"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
+<a id="__codelineno-0-239" name="__codelineno-0-239"></a>    <span class="c1"># counter byte length encoding</span>
+<a id="__codelineno-0-240" name="__codelineno-0-240"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ID</span><span class="p">:</span>
+<a id="__codelineno-0-241" name="__codelineno-0-241"></a>        <span class="k">if</span> <span class="mi">64</span> <span class="o">&lt;=</span> <span class="n">length</span> <span class="o">&lt;=</span> <span class="mi">96</span><span class="p">:</span>
+<a id="__codelineno-0-242" name="__codelineno-0-242"></a>            <span class="k">return</span> <span class="p">(</span><span class="n">length</span> <span class="o">-</span> <span class="mi">64</span><span class="p">)</span> <span class="o">//</span> <span class="mi">8</span>
+<a id="__codelineno-0-243" name="__codelineno-0-243"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
+<a id="__codelineno-0-244" name="__codelineno-0-244"></a>    <span class="k">else</span><span class="p">:</span>
+<a id="__codelineno-0-245" name="__codelineno-0-245"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2478,7 +2478,8 @@ <h3 id="iscc_core.codec.decode_length" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-249">249</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-248">248</a></span>
+<span class="normal"><a href="#__codelineno-0-249">249</a></span>
 <span class="normal"><a href="#__codelineno-0-250">250</a></span>
 <span class="normal"><a href="#__codelineno-0-251">251</a></span>
 <span class="normal"><a href="#__codelineno-0-252">252</a></span>
@@ -2492,23 +2493,22 @@ <h3 id="iscc_core.codec.decode_length" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-260">260</a></span>
 <span class="normal"><a href="#__codelineno-0-261">261</a></span>
 <span class="normal"><a href="#__codelineno-0-262">262</a></span>
-<span class="normal"><a href="#__codelineno-0-263">263</a></span>
-<span class="normal"><a href="#__codelineno-0-264">264</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-249" name="__codelineno-0-249"></a><span class="k">def</span> <span class="nf">decode_length</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">length</span><span class="p">):</span>
-<a id="__codelineno-0-250" name="__codelineno-0-250"></a>    <span class="c1"># type: (MainType, Length) -&gt; LN</span>
-<a id="__codelineno-0-251" name="__codelineno-0-251"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-252" name="__codelineno-0-252"></a><span class="sd">    Dedoce raw length value from ISCC header to length of digest in number of bits.</span>
-<a id="__codelineno-0-253" name="__codelineno-0-253"></a>
-<a id="__codelineno-0-254" name="__codelineno-0-254"></a><span class="sd">    Decodes a raw header integer value in to its semantically meaningfull value (e.g.</span>
-<a id="__codelineno-0-255" name="__codelineno-0-255"></a><span class="sd">    number of bits)</span>
-<a id="__codelineno-0-256" name="__codelineno-0-256"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-257" name="__codelineno-0-257"></a>    <span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">(</span><span class="n">MT</span><span class="o">.</span><span class="n">META</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">SEMANTIC</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">CONTENT</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">DATA</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">INSTANCE</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">FLAKE</span><span class="p">):</span>
-<a id="__codelineno-0-258" name="__codelineno-0-258"></a>        <span class="k">return</span> <span class="n">LN</span><span class="p">((</span><span class="n">length</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">32</span><span class="p">)</span>
-<a id="__codelineno-0-259" name="__codelineno-0-259"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ISCC</span><span class="p">:</span>
-<a id="__codelineno-0-260" name="__codelineno-0-260"></a>        <span class="k">return</span> <span class="n">LN</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">decode_units</span><span class="p">(</span><span class="n">length</span><span class="p">))</span> <span class="o">*</span> <span class="mi">64</span> <span class="o">+</span> <span class="mi">128</span><span class="p">)</span>
-<a id="__codelineno-0-261" name="__codelineno-0-261"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ID</span><span class="p">:</span>
-<a id="__codelineno-0-262" name="__codelineno-0-262"></a>        <span class="k">return</span> <span class="n">LN</span><span class="p">(</span><span class="n">length</span> <span class="o">*</span> <span class="mi">8</span> <span class="o">+</span> <span class="mi">64</span><span class="p">)</span>
-<a id="__codelineno-0-263" name="__codelineno-0-263"></a>    <span class="k">else</span><span class="p">:</span>
-<a id="__codelineno-0-264" name="__codelineno-0-264"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Invalid length </span><span class="si">{</span><span class="n">length</span><span class="si">}</span><span class="s2"> for MainType </span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-263">263</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-248" name="__codelineno-0-248"></a><span class="k">def</span> <span class="nf">decode_length</span><span class="p">(</span><span class="n">mtype</span><span class="p">,</span> <span class="n">length</span><span class="p">):</span>
+<a id="__codelineno-0-249" name="__codelineno-0-249"></a>    <span class="c1"># type: (MainType, Length) -&gt; LN</span>
+<a id="__codelineno-0-250" name="__codelineno-0-250"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-251" name="__codelineno-0-251"></a><span class="sd">    Dedoce raw length value from ISCC header to length of digest in number of bits.</span>
+<a id="__codelineno-0-252" name="__codelineno-0-252"></a>
+<a id="__codelineno-0-253" name="__codelineno-0-253"></a><span class="sd">    Decodes a raw header integer value in to its semantically meaningfull value (e.g.</span>
+<a id="__codelineno-0-254" name="__codelineno-0-254"></a><span class="sd">    number of bits)</span>
+<a id="__codelineno-0-255" name="__codelineno-0-255"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-256" name="__codelineno-0-256"></a>    <span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">(</span><span class="n">MT</span><span class="o">.</span><span class="n">META</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">SEMANTIC</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">CONTENT</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">DATA</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">INSTANCE</span><span class="p">,</span> <span class="n">MT</span><span class="o">.</span><span class="n">FLAKE</span><span class="p">):</span>
+<a id="__codelineno-0-257" name="__codelineno-0-257"></a>        <span class="k">return</span> <span class="n">LN</span><span class="p">((</span><span class="n">length</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">32</span><span class="p">)</span>
+<a id="__codelineno-0-258" name="__codelineno-0-258"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ISCC</span><span class="p">:</span>
+<a id="__codelineno-0-259" name="__codelineno-0-259"></a>        <span class="k">return</span> <span class="n">LN</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">decode_units</span><span class="p">(</span><span class="n">length</span><span class="p">))</span> <span class="o">*</span> <span class="mi">64</span> <span class="o">+</span> <span class="mi">128</span><span class="p">)</span>
+<a id="__codelineno-0-260" name="__codelineno-0-260"></a>    <span class="k">elif</span> <span class="n">mtype</span> <span class="o">==</span> <span class="n">MT</span><span class="o">.</span><span class="n">ID</span><span class="p">:</span>
+<a id="__codelineno-0-261" name="__codelineno-0-261"></a>        <span class="k">return</span> <span class="n">LN</span><span class="p">(</span><span class="n">length</span> <span class="o">*</span> <span class="mi">8</span> <span class="o">+</span> <span class="mi">64</span><span class="p">)</span>
+<a id="__codelineno-0-262" name="__codelineno-0-262"></a>    <span class="k">else</span><span class="p">:</span>
+<a id="__codelineno-0-263" name="__codelineno-0-263"></a>        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Invalid length </span><span class="si">{</span><span class="n">length</span><span class="si">}</span><span class="s2"> for MainType </span><span class="si">{</span><span class="n">mtype</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2536,17 +2536,17 @@ <h3 id="iscc_core.codec.encode_base32" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-267">267</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-266">266</a></span>
+<span class="normal"><a href="#__codelineno-0-267">267</a></span>
 <span class="normal"><a href="#__codelineno-0-268">268</a></span>
 <span class="normal"><a href="#__codelineno-0-269">269</a></span>
 <span class="normal"><a href="#__codelineno-0-270">270</a></span>
-<span class="normal"><a href="#__codelineno-0-271">271</a></span>
-<span class="normal"><a href="#__codelineno-0-272">272</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-267" name="__codelineno-0-267"></a><span class="k">def</span> <span class="nf">encode_base32</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
-<a id="__codelineno-0-268" name="__codelineno-0-268"></a>    <span class="c1"># type: (bytes) -&gt; str</span>
-<a id="__codelineno-0-269" name="__codelineno-0-269"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-270" name="__codelineno-0-270"></a><span class="sd">    Standard RFC4648 base32 encoding without padding.</span>
-<a id="__codelineno-0-271" name="__codelineno-0-271"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-272" name="__codelineno-0-272"></a>    <span class="k">return</span> <span class="n">b32encode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;ascii&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;=&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-271">271</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-266" name="__codelineno-0-266"></a><span class="k">def</span> <span class="nf">encode_base32</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
+<a id="__codelineno-0-267" name="__codelineno-0-267"></a>    <span class="c1"># type: (bytes) -&gt; str</span>
+<a id="__codelineno-0-268" name="__codelineno-0-268"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-269" name="__codelineno-0-269"></a><span class="sd">    Standard RFC4648 base32 encoding without padding.</span>
+<a id="__codelineno-0-270" name="__codelineno-0-270"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-271" name="__codelineno-0-271"></a>    <span class="k">return</span> <span class="n">b32encode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;ascii&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;=&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2574,7 +2574,8 @@ <h3 id="iscc_core.codec.decode_base32" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-275">275</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-274">274</a></span>
+<span class="normal"><a href="#__codelineno-0-275">275</a></span>
 <span class="normal"><a href="#__codelineno-0-276">276</a></span>
 <span class="normal"><a href="#__codelineno-0-277">277</a></span>
 <span class="normal"><a href="#__codelineno-0-278">278</a></span>
@@ -2582,17 +2583,16 @@ <h3 id="iscc_core.codec.decode_base32" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-280">280</a></span>
 <span class="normal"><a href="#__codelineno-0-281">281</a></span>
 <span class="normal"><a href="#__codelineno-0-282">282</a></span>
-<span class="normal"><a href="#__codelineno-0-283">283</a></span>
-<span class="normal"><a href="#__codelineno-0-284">284</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-275" name="__codelineno-0-275"></a><span class="k">def</span> <span class="nf">decode_base32</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
-<a id="__codelineno-0-276" name="__codelineno-0-276"></a>    <span class="c1"># type: (str) -&gt; bytes</span>
-<a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="sd">    Standard RFC4648 base32 decoding without padding and with casefolding.</span>
-<a id="__codelineno-0-279" name="__codelineno-0-279"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-280" name="__codelineno-0-280"></a>    <span class="c1"># python stdlib does not support base32 without padding, so we have to re-pad.</span>
-<a id="__codelineno-0-281" name="__codelineno-0-281"></a>    <span class="n">cl</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span>
-<a id="__codelineno-0-282" name="__codelineno-0-282"></a>    <span class="n">pad_length</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">ceil</span><span class="p">(</span><span class="n">cl</span> <span class="o">/</span> <span class="mi">8</span><span class="p">)</span> <span class="o">*</span> <span class="mi">8</span> <span class="o">-</span> <span class="n">cl</span>
-<a id="__codelineno-0-283" name="__codelineno-0-283"></a>
-<a id="__codelineno-0-284" name="__codelineno-0-284"></a>    <span class="k">return</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">b32decode</span><span class="p">(</span><span class="n">code</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="n">pad_length</span><span class="p">,</span> <span class="n">casefold</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
+<span class="normal"><a href="#__codelineno-0-283">283</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-274" name="__codelineno-0-274"></a><span class="k">def</span> <span class="nf">decode_base32</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
+<a id="__codelineno-0-275" name="__codelineno-0-275"></a>    <span class="c1"># type: (str) -&gt; bytes</span>
+<a id="__codelineno-0-276" name="__codelineno-0-276"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-277" name="__codelineno-0-277"></a><span class="sd">    Standard RFC4648 base32 decoding without padding and with casefolding.</span>
+<a id="__codelineno-0-278" name="__codelineno-0-278"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-279" name="__codelineno-0-279"></a>    <span class="c1"># python stdlib does not support base32 without padding, so we have to re-pad.</span>
+<a id="__codelineno-0-280" name="__codelineno-0-280"></a>    <span class="n">cl</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span>
+<a id="__codelineno-0-281" name="__codelineno-0-281"></a>    <span class="n">pad_length</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">ceil</span><span class="p">(</span><span class="n">cl</span> <span class="o">/</span> <span class="mi">8</span><span class="p">)</span> <span class="o">*</span> <span class="mi">8</span> <span class="o">-</span> <span class="n">cl</span>
+<a id="__codelineno-0-282" name="__codelineno-0-282"></a>
+<a id="__codelineno-0-283" name="__codelineno-0-283"></a>    <span class="k">return</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">b32decode</span><span class="p">(</span><span class="n">code</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="n">pad_length</span><span class="p">,</span> <span class="n">casefold</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2952,19 +2952,19 @@ <h3 id="iscc_core.codec.encode_base64" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-287">287</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-286">286</a></span>
+<span class="normal"><a href="#__codelineno-0-287">287</a></span>
 <span class="normal"><a href="#__codelineno-0-288">288</a></span>
 <span class="normal"><a href="#__codelineno-0-289">289</a></span>
 <span class="normal"><a href="#__codelineno-0-290">290</a></span>
 <span class="normal"><a href="#__codelineno-0-291">291</a></span>
-<span class="normal"><a href="#__codelineno-0-292">292</a></span>
-<span class="normal"><a href="#__codelineno-0-293">293</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-287" name="__codelineno-0-287"></a><span class="k">def</span> <span class="nf">encode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
-<a id="__codelineno-0-288" name="__codelineno-0-288"></a>    <span class="c1"># type: (bytes) -&gt; str</span>
-<a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">    Standard RFC4648 base64url encoding without padding.</span>
-<a id="__codelineno-0-291" name="__codelineno-0-291"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-292" name="__codelineno-0-292"></a>    <span class="n">code</span> <span class="o">=</span> <span class="n">urlsafe_b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;ascii&quot;</span><span class="p">)</span>
-<a id="__codelineno-0-293" name="__codelineno-0-293"></a>    <span class="k">return</span> <span class="n">code</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;=&quot;</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-292">292</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-286" name="__codelineno-0-286"></a><span class="k">def</span> <span class="nf">encode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
+<a id="__codelineno-0-287" name="__codelineno-0-287"></a>    <span class="c1"># type: (bytes) -&gt; str</span>
+<a id="__codelineno-0-288" name="__codelineno-0-288"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-289" name="__codelineno-0-289"></a><span class="sd">    Standard RFC4648 base64url encoding without padding.</span>
+<a id="__codelineno-0-290" name="__codelineno-0-290"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-291" name="__codelineno-0-291"></a>    <span class="n">code</span> <span class="o">=</span> <span class="n">urlsafe_b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;ascii&quot;</span><span class="p">)</span>
+<a id="__codelineno-0-292" name="__codelineno-0-292"></a>    <span class="k">return</span> <span class="n">code</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;=&quot;</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -2992,21 +2992,21 @@ <h3 id="iscc_core.codec.decode_base64" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-296">296</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-295">295</a></span>
+<span class="normal"><a href="#__codelineno-0-296">296</a></span>
 <span class="normal"><a href="#__codelineno-0-297">297</a></span>
 <span class="normal"><a href="#__codelineno-0-298">298</a></span>
 <span class="normal"><a href="#__codelineno-0-299">299</a></span>
 <span class="normal"><a href="#__codelineno-0-300">300</a></span>
 <span class="normal"><a href="#__codelineno-0-301">301</a></span>
-<span class="normal"><a href="#__codelineno-0-302">302</a></span>
-<span class="normal"><a href="#__codelineno-0-303">303</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-296" name="__codelineno-0-296"></a><span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
-<a id="__codelineno-0-297" name="__codelineno-0-297"></a>    <span class="c1"># type: (str) -&gt; bytes</span>
-<a id="__codelineno-0-298" name="__codelineno-0-298"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-299" name="__codelineno-0-299"></a><span class="sd">    Standard RFC4648 base64url decoding without padding.</span>
-<a id="__codelineno-0-300" name="__codelineno-0-300"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-301" name="__codelineno-0-301"></a>    <span class="n">padding</span> <span class="o">=</span> <span class="mi">4</span> <span class="o">-</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span><span class="p">)</span>
-<a id="__codelineno-0-302" name="__codelineno-0-302"></a>    <span class="n">string</span> <span class="o">=</span> <span class="n">code</span> <span class="o">+</span> <span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="n">padding</span><span class="p">)</span>
-<a id="__codelineno-0-303" name="__codelineno-0-303"></a>    <span class="k">return</span> <span class="n">urlsafe_b64decode</span><span class="p">(</span><span class="n">string</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-302">302</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-295" name="__codelineno-0-295"></a><span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
+<a id="__codelineno-0-296" name="__codelineno-0-296"></a>    <span class="c1"># type: (str) -&gt; bytes</span>
+<a id="__codelineno-0-297" name="__codelineno-0-297"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-298" name="__codelineno-0-298"></a><span class="sd">    Standard RFC4648 base64url decoding without padding.</span>
+<a id="__codelineno-0-299" name="__codelineno-0-299"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-300" name="__codelineno-0-300"></a>    <span class="n">padding</span> <span class="o">=</span> <span class="mi">4</span> <span class="o">-</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span><span class="p">)</span>
+<a id="__codelineno-0-301" name="__codelineno-0-301"></a>    <span class="n">string</span> <span class="o">=</span> <span class="n">code</span> <span class="o">+</span> <span class="p">(</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="n">padding</span><span class="p">)</span>
+<a id="__codelineno-0-302" name="__codelineno-0-302"></a>    <span class="k">return</span> <span class="n">urlsafe_b64decode</span><span class="p">(</span><span class="n">string</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -3035,23 +3035,23 @@ <h3 id="iscc_core.codec.encode_base32hex" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-306">306</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-305">305</a></span>
+<span class="normal"><a href="#__codelineno-0-306">306</a></span>
 <span class="normal"><a href="#__codelineno-0-307">307</a></span>
 <span class="normal"><a href="#__codelineno-0-308">308</a></span>
 <span class="normal"><a href="#__codelineno-0-309">309</a></span>
 <span class="normal"><a href="#__codelineno-0-310">310</a></span>
 <span class="normal"><a href="#__codelineno-0-311">311</a></span>
 <span class="normal"><a href="#__codelineno-0-312">312</a></span>
-<span class="normal"><a href="#__codelineno-0-313">313</a></span>
-<span class="normal"><a href="#__codelineno-0-314">314</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-306" name="__codelineno-0-306"></a><span class="k">def</span> <span class="nf">encode_base32hex</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
-<a id="__codelineno-0-307" name="__codelineno-0-307"></a>    <span class="c1"># type: (bytes) -&gt;  str</span>
-<a id="__codelineno-0-308" name="__codelineno-0-308"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-309" name="__codelineno-0-309"></a><span class="sd">    RFC4648 Base32hex encoding without padding</span>
-<a id="__codelineno-0-310" name="__codelineno-0-310"></a>
-<a id="__codelineno-0-311" name="__codelineno-0-311"></a><span class="sd">    see: https://tools.ietf.org/html/rfc4648#page-10</span>
-<a id="__codelineno-0-312" name="__codelineno-0-312"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="n">b32</span> <span class="o">=</span> <span class="n">encode_base32</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
-<a id="__codelineno-0-314" name="__codelineno-0-314"></a>    <span class="k">return</span> <span class="n">b32</span><span class="o">.</span><span class="n">translate</span><span class="p">(</span><span class="n">b32_to_hex</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-313">313</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-305" name="__codelineno-0-305"></a><span class="k">def</span> <span class="nf">encode_base32hex</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
+<a id="__codelineno-0-306" name="__codelineno-0-306"></a>    <span class="c1"># type: (bytes) -&gt;  str</span>
+<a id="__codelineno-0-307" name="__codelineno-0-307"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-308" name="__codelineno-0-308"></a><span class="sd">    RFC4648 Base32hex encoding without padding</span>
+<a id="__codelineno-0-309" name="__codelineno-0-309"></a>
+<a id="__codelineno-0-310" name="__codelineno-0-310"></a><span class="sd">    see: https://tools.ietf.org/html/rfc4648#page-10</span>
+<a id="__codelineno-0-311" name="__codelineno-0-311"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-312" name="__codelineno-0-312"></a>    <span class="n">b32</span> <span class="o">=</span> <span class="n">encode_base32</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+<a id="__codelineno-0-313" name="__codelineno-0-313"></a>    <span class="k">return</span> <span class="n">b32</span><span class="o">.</span><span class="n">translate</span><span class="p">(</span><span class="n">b32_to_hex</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
   </div>
@@ -3080,7 +3080,8 @@ <h3 id="iscc_core.codec.decode_base32hex" class="doc doc-heading">
 
       <details class="quote">
         <summary>Source code in <code>iscc_core\codec.py</code></summary>
-        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-317">317</a></span>
+        <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-316">316</a></span>
+<span class="normal"><a href="#__codelineno-0-317">317</a></span>
 <span class="normal"><a href="#__codelineno-0-318">318</a></span>
 <span class="normal"><a href="#__codelineno-0-319">319</a></span>
 <span class="normal"><a href="#__codelineno-0-320">320</a></span>
@@ -3088,14 +3089,15 @@ <h3 id="iscc_core.codec.decode_base32hex" class="doc doc-heading">
 <span class="normal"><a href="#__codelineno-0-322">322</a></span>
 <span class="normal"><a href="#__codelineno-0-323">323</a></span>
 <span class="normal"><a href="#__codelineno-0-324">324</a></span>
-<span class="normal"><a href="#__codelineno-0-325">325</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-317" name="__codelineno-0-317"></a><span class="k">def</span> <span class="nf">decode_base32hex</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
-<a id="__codelineno-0-318" name="__codelineno-0-318"></a>    <span class="c1"># type: (str) -&gt; bytes</span>
-<a id="__codelineno-0-319" name="__codelineno-0-319"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<a id="__codelineno-0-320" name="__codelineno-0-320"></a><span class="sd">    RFC4648 Base32hex decoding without padding</span>
-<a id="__codelineno-0-321" name="__codelineno-0-321"></a>
-<a id="__codelineno-0-322" name="__codelineno-0-322"></a><span class="sd">    see: https://tools.ietf.org/html/rfc4648#page-10</span>
-<a id="__codelineno-0-323" name="__codelineno-0-323"></a><span class="sd">    &quot;&quot;&quot;</span>
-<a id="__codelineno-0-324" name="__codelineno-0-324"></a>    <span class="n">b32</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">translate</span><span class="p">(</span><span class="n">hex_to_b32</span><span class="p">)</span>
+<span class="normal"><a href="#__codelineno-0-325">325</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-316" name="__codelineno-0-316"></a><span class="k">def</span> <span class="nf">decode_base32hex</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
+<a id="__codelineno-0-317" name="__codelineno-0-317"></a>    <span class="c1"># type: (str) -&gt; bytes</span>
+<a id="__codelineno-0-318" name="__codelineno-0-318"></a><span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<a id="__codelineno-0-319" name="__codelineno-0-319"></a><span class="sd">    RFC4648 Base32hex decoding without padding</span>
+<a id="__codelineno-0-320" name="__codelineno-0-320"></a>
+<a id="__codelineno-0-321" name="__codelineno-0-321"></a><span class="sd">    see: https://tools.ietf.org/html/rfc4648#page-10</span>
+<a id="__codelineno-0-322" name="__codelineno-0-322"></a><span class="sd">    &quot;&quot;&quot;</span>
+<a id="__codelineno-0-323" name="__codelineno-0-323"></a>    <span class="c1"># Make sure we use upper-case version for translation</span>
+<a id="__codelineno-0-324" name="__codelineno-0-324"></a>    <span class="n">b32</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="o">.</span><span class="n">translate</span><span class="p">(</span><span class="n">hex_to_b32</span><span class="p">)</span>
 <a id="__codelineno-0-325" name="__codelineno-0-325"></a>    <span class="k">return</span> <span class="n">decode_base32</span><span class="p">(</span><span class="n">b32</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
       </details>
@@ -3600,10 +3602,10 @@ <h3 id="iscc_core.codec.iscc_validate" class="doc doc-heading">
 <a id="__codelineno-0-511" name="__codelineno-0-511"></a><span class="sd">    &quot;&quot;&quot;</span>
 <a id="__codelineno-0-512" name="__codelineno-0-512"></a>
 <a id="__codelineno-0-513" name="__codelineno-0-513"></a>    <span class="c1"># Basic regex validation</span>
-<a id="__codelineno-0-514" name="__codelineno-0-514"></a>    <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s2">&quot;^ISCC:[A-Z2-7]{10,60}$&quot;</span><span class="p">,</span> <span class="n">iscc</span><span class="p">)</span>
+<a id="__codelineno-0-514" name="__codelineno-0-514"></a>    <span class="n">match</span> <span class="o">=</span> <span class="n">CANONICAL_REGEX</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">iscc</span><span class="p">)</span>
 <a id="__codelineno-0-515" name="__codelineno-0-515"></a>    <span class="k">if</span> <span class="ow">not</span> <span class="n">match</span><span class="p">:</span>
 <a id="__codelineno-0-516" name="__codelineno-0-516"></a>        <span class="k">if</span> <span class="n">strict</span><span class="p">:</span>
-<a id="__codelineno-0-517" name="__codelineno-0-517"></a>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;ISCC string does not match ^ISCC:[A-Z2-7]{10,60}$&quot;</span><span class="p">)</span>
+<a id="__codelineno-0-517" name="__codelineno-0-517"></a>            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;ISCC string does not match ^ISCC:[A-Z2-7]{10,68}$&quot;</span><span class="p">)</span>
 <a id="__codelineno-0-518" name="__codelineno-0-518"></a>        <span class="k">else</span><span class="p">:</span>
 <a id="__codelineno-0-519" name="__codelineno-0-519"></a>            <span class="k">return</span> <span class="kc">False</span>
 <a id="__codelineno-0-520" name="__codelineno-0-520"></a>
diff --git a/search/search_index.json b/search/search_index.json
index 7ba0169..273181f 100755
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"ISCC - Codec &amp; Algorithms","text":"<p><code>iscc-core</code> is the reference implementation of the core algorithms of the ISCC (International Standard Content Code)</p>"},{"location":"#what-is-the-iscc","title":"What is the ISCC","text":"<p>The ISCC is a similarity preserving fingerprint and identifier for digital media assets.</p> <p>ISCCs are generated algorithmically from digital content, just like cryptographic hashes. However, instead of using a single cryptographic hash function to identify data only, the ISCC uses various algorithms to create a composite identifier that exhibits similarity-preserving properties (soft hash).</p> <p>The component-based structure of the ISCC identifies content at multiple levels of abstraction. Each component is self-describing, modular, and can be used separately or with others to aid in various content identification tasks. The algorithmic design supports content deduplication, database synchronization, indexing, integrity verification, timestamping, versioning, data provenance, similarity clustering, anomaly detection, usage tracking, allocation of royalties, fact-checking and general digital asset management use-cases.</p>"},{"location":"#what-is-iscc-core","title":"What is <code>iscc-core</code>","text":"<p><code>iscc-core</code> is the python based reference implementation of the ISCC core algorithms as defined by ISO 24138. It also a good reference for porting ISCC to other programming languages.</p> <p>Tip</p> <p>This is a low level reference implementation that does not inlcude features like mediatype detection, metadata extraction or file format specific content extraction. Please have a look at iscc-sdk which adds those higher level features on top of the <code>iscc-core</code> library.</p>"},{"location":"#implementors-guide","title":"Implementors Guide","text":""},{"location":"#reproducible-environment","title":"Reproducible Environment","text":"<p>For reproducible installation of the reference implementation we included a <code>poetry.lock</code> file with pinned dependencies. Install them using Python Poetry with the command <code>poetry install</code> in the root folder.</p>"},{"location":"#repository-structure","title":"Repository structure","text":"<pre><code>iscc-core\n\u251c\u2500\u2500 docs       # Markdown and other assets for mkdocs documentation\n\u251c\u2500\u2500 examples   # Example scripts using the reference code\n\u251c\u2500\u2500 iscc_core  # Actual source code of the reference implementation\n\u251c\u2500\u2500 tests      # Tests for the reference implementation\n\u2514\u2500\u2500 tools      # Development tools\n</code></pre>"},{"location":"#testing-conformance","title":"Testing &amp; Conformance","text":"<p>The reference implementation comes with 100% test coverage. To run the conformance selftest from the repository root use <code>poetry run python -m iscc_core</code>. To run the complete test suite use <code>poetry run pytest</code>.</p> <p>To build a conformant implementation work through the follwing top level entrypoint functions:</p> <pre><code>gen_meta_code_v0\ngen_text_code_v0\ngen_image_code_v0\ngen_audio_code_v0\ngen_video_code_v0\ngen_mixed_code_v0\ngen_data_code_v0\ngen_instance_code_v0\ngen_iscc_code_v0\n</code></pre> <p>The corresponding test vectors can be found in <code>iscc_core/data.json</code>.</p>"},{"location":"#iscc-architecture","title":"ISCC Architecture","text":""},{"location":"#iscc-maintypes","title":"ISCC MainTypes","text":"Idx Slug Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more components with common header"},{"location":"#installation","title":"Installation","text":"<p>Use the package manager pip to install <code>iscc-core</code> as a library.</p> <pre><code>pip install iscc-core\n</code></pre>"},{"location":"#quick-start","title":"Quick Start","text":"<pre><code>import json\nimport iscc_core as ic\nmeta_code = ic.gen_meta_code(name=\"ISCC Test Document!\")\nprint(f\"Meta-Code:     {meta_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(meta_code['iscc'])}\\n\")\n# Extract text from file\nwith open(\"demo.txt\", \"rt\", encoding=\"utf-8\") as stream:\ntext = stream.read()\ntext_code = ic.gen_text_code_v0(text)\nprint(f\"Text-Code:     {text_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(text_code['iscc'])}\\n\")\n# Process raw bytes of textfile\nwith open(\"demo.txt\", \"rb\") as stream:\ndata_code = ic.gen_data_code(stream)\nprint(f\"Data-Code:     {data_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(data_code['iscc'])}\\n\")\nstream.seek(0)\ninstance_code = ic.gen_instance_code(stream)\nprint(f\"Instance-Code: {instance_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(instance_code['iscc'])}\\n\")\n# Combine ISCC-UNITs into ISCC-CODE\niscc_code = ic.gen_iscc_code(\n(meta_code[\"iscc\"], text_code[\"iscc\"], data_code[\"iscc\"], instance_code[\"iscc\"])\n)\n# Create convenience `Code` object from ISCC string\niscc_obj = ic.Code(iscc_code[\"iscc\"])\nprint(f\"ISCC-CODE:     {ic.iscc_normalize(iscc_obj.code)}\")\nprint(f\"Structure:     {iscc_obj.explain}\")\nprint(f\"Multiformat:   {iscc_obj.mf_base32}\\n\")\n# Compare with changed ISCC-CODE:\nnew_dc, new_ic = ic.Code.rnd(mt=ic.MT.DATA), ic.Code.rnd(mt=ic.MT.INSTANCE)\nnew_iscc = ic.gen_iscc_code((meta_code[\"iscc\"], text_code[\"iscc\"], new_dc.uri, new_ic.uri))\nprint(f\"Compare ISCC-CODES:\\n{iscc_obj.uri}\\n{new_iscc['iscc']}\")\nprint(json.dumps(ic.iscc_compare(iscc_obj.code, new_iscc[\"iscc\"]), indent=2))\n</code></pre> <p>The output of this example is as follows:</p> <pre><code>Meta-Code:     ISCC:AAAT4EBWK27737D2\nStructure:     META-NONE-V0-64-3e103656bffdfc7a\n\nText-Code:     ISCC:EAAQMBEYQF6457DP\nStructure:     CONTENT-TEXT-V0-64-060498817dcefc6f\n\nData-Code:     ISCC:GAA7UJMLDXHPPENG\nStructure:     DATA-NONE-V0-64-fa258b1dcef791a6\n\nInstance-Code: ISCC:IAA3Y7HR2FEZCU4N\nStructure:     INSTANCE-NONE-V0-64-bc7cf1d14991538d\n\nISCC-CODE:     ISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nStructure:     ISCC-TEXT-V0-MCDI-3e103656bffdfc7a060498817dcefc6ffa258b1dcef791a6bc7cf1d14991538d\nMultiformat:   bzqavabj6ca3fnp757r5ambeyqf6457dp7isywhoo66i2npd46hiutektru\n\nCompare ISCC-CODES:\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G7Y7HA2BMECKMVRBEQXR2BJOS6NA\n{\n  \"meta_dist\": 0,\n  \"content_dist\": 0,\n  \"data_dist\": 33,\n  \"instance_match\": false\n}\n</code></pre>"},{"location":"#documentation","title":"Documentation","text":"<p>Documentation is published at https://core.iscc.codes</p>"},{"location":"#development","title":"Development","text":"<p>Requirements</p> <ul> <li>Python 3.7.2 or higher for code generation and static site building.</li> <li>Poetry for installation and dependency management.</li> </ul> <p>Development Setup</p> <pre><code>git clone https://github.com/iscc/iscc-core.git\ncd iscc-core\npoetry install\n</code></pre> <p>Development Tasks</p> <p>Tests, coverage, code formatting and other tasks can be run with the <code>poe</code> command:</p> <pre><code>poe\n\nPoe the Poet - A task runner that works well with poetry.\nversion 0.18.1\n\nResult: No task specified.\n\nUSAGE\n  poe [-h] [-v | -q] [--root PATH] [--ansi | --no-ansi] task [task arguments]\nGLOBAL OPTIONS\n  -h, --help     Show this help page and exit\n--version      Print the version and exit\n-v, --verbose  Increase command output (repeatable)\n-q, --quiet    Decrease command output (repeatable)\n-d, --dry-run  Print the task contents but don't actually run it\n  --root PATH    Specify where to find the pyproject.toml\n  --ansi         Force enable ANSI output\n  --no-ansi      Force disable ANSI output\nCONFIGURED TASKS\n  gentests       Generate conformance test data\n  format         Code style formating with black\n  docs           Copy README.md to /docs\n  format-md      Markdown formating with mdformat\n  lf             Convert line endings to lf\n  test           Run tests with coverage\n  sec            Security check with bandit\n  all\n</code></pre> <p>Use <code>poe all</code> to run all tasks before committing any changes.</p>"},{"location":"#maintainers","title":"Maintainers","text":"<p>@titusz</p>"},{"location":"#contributing","title":"Contributing","text":"<p>Pull requests are welcome. For significant changes, please open an issue first to discuss your plans. Please make sure to update tests as appropriate.</p> <p>You may also want join our developer chat on Telegram at https://t.me/iscc_dev.</p>"},{"location":"changelog/","title":"Changelog","text":""},{"location":"changelog/#108-2024-01-30","title":"[1.0.8] - 2024-01-30","text":"<ul> <li>Added implementors guide to README.md</li> <li>Improved pydantic v2 import logic</li> <li>Improved prefix extraction during normalization</li> <li>Improved canonical ISCC string validation</li> </ul>"},{"location":"changelog/#107-2024-01-07","title":"[1.0.7] - 2024-01-07","text":"<ul> <li>Support pydantic v1 &amp; v2</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#106-2023-12-15","title":"[1.0.6] - 2023-12-15","text":"<ul> <li>Added wheel package testing to CI</li> <li>Fixed incompatible dependencies</li> </ul>"},{"location":"changelog/#105-2023-12-07","title":"[1.0.5] - 2023-12-07","text":"<ul> <li>Improved simhash performance</li> <li>Added native dct and wtahash support</li> <li>Added Python 3.12 support</li> <li>Updated and relax dependencies</li> </ul>"},{"location":"changelog/#104-2023-06-05","title":"[1.0.4] - 2023-06-05","text":"<ul> <li>Removed bases dependency</li> <li>Fixed mkdocstrings</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#103-2023-03-12","title":"[1.0.3] - 2023-03-12","text":"<ul> <li>Fix binary wheels</li> </ul>"},{"location":"changelog/#102-2023-03-12","title":"[1.0.2] - 2023-03-12","text":"<ul> <li>Publish binary wheels</li> </ul>"},{"location":"changelog/#101-2023-03-11","title":"[1.0.1] - 2023-03-11","text":"<ul> <li>Switch to standard bitarray module</li> <li>Switch tests to latest environments</li> <li>Add Python 3.11 support to TROVE classifiers</li> </ul>"},{"location":"changelog/#100-2023-01-24","title":"[1.0.0] - 2023-01-24","text":"<ul> <li>ISO/CD 24138 v1 Release</li> <li>Updateted dependencies</li> </ul>"},{"location":"changelog/#0214-2023-01-17","title":"[0.2.14] - 2023-01-17","text":"<ul> <li>Added ISO Reference documentation</li> <li>Removed non-standard conformance tests</li> <li>Fixed Meta-Code documentation</li> <li>Improved documentation CSS</li> </ul>"},{"location":"changelog/#0213-2023-01-16","title":"[0.2.13] - 2023-01-16","text":"<ul> <li>Added documentation for options</li> <li>Added python 3.11 support</li> <li>Added Markdown formating</li> <li>Added developmnet documentation</li> <li>Updated architecture figures</li> <li>Documentation cleanup</li> <li>Improved example code</li> <li>Optimized LF conversion</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0212-2022-11-24","title":"[0.2.12] - 2022-11-24","text":"<ul> <li>Fixed issue with data url compound media types</li> <li>Added ISCC version validation to <code>iscc_validate</code></li> <li>Added prefix check to <code>iscc_normalize</code></li> <li>Bundled fonts with documentation</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0211-2022-07-03","title":"[0.2.11] - 2022-07-03","text":"<ul> <li>Add support for gracefull build failures</li> </ul>"},{"location":"changelog/#0210-2022-07-03","title":"[0.2.10] - 2022-07-03","text":"<ul> <li>Fix pip instalation is missing setuptools</li> <li>Update mkdocs</li> </ul>"},{"location":"changelog/#029-2022-07-03","title":"[0.2.9] - 2022-07-03","text":"<ul> <li>Added iscc_compare function</li> <li>Optimized soft_hash_audio performance</li> <li>Removed Cython from build requirements</li> <li>Fixed api listing tool</li> <li>Updated codec architecture figure</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#028-2022-04-21","title":"[0.2.8] - 2022-04-21","text":"<ul> <li>Fixed bug with subtype for semantic code</li> <li>Changed URI representation to upper case</li> <li>Changed to disallow ISCC-ID creation from ISCC-IDs</li> <li>Added line conversion tool</li> <li>Removed source wheel distribution</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#027-2022-04-16","title":"[0.2.7] - 2022-04-16","text":"<ul> <li>Fixed bug in iscc_id_incr_v0</li> <li>Added support to accept ISCC-ID URI as input for iscc_id_incr_v0</li> <li>Added guard against custom subtype in random ISCC-CODE generation.</li> </ul>"},{"location":"changelog/#026-2022-04-13","title":"[0.2.6] - 2022-04-13","text":"<ul> <li>Added <code>KY</code> and <code>MM</code> to valid prefixes</li> <li>Added support to check for compiled extension modules</li> <li>Added universal wheel distribution</li> </ul>"},{"location":"changelog/#025-2022-04-10","title":"[0.2.5] - 2022-04-10","text":"<ul> <li>Fixed missing <code>jcs</code> dependency</li> <li>Added SubType <code>NONE</code> to MT.ISCC to distinquish from SUM</li> <li>Added support for deterministic generation of random ISCC-CODEs</li> <li>Added support for custom bit-sizes for random ISCC-CODEs</li> <li>Moved changelog into separate file</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#024-2022-03-19","title":"[0.2.4] - 2022-03-19","text":"<ul> <li>Updated dependencies</li> <li>Added Flake.from_int and Flake.from_string</li> <li>Made Flake comparable and hashable</li> <li>Use standard hex encoded multihash for datahash and metahash</li> </ul>"},{"location":"changelog/#023-2022-03-06","title":"[0.2.3] - 2022-03-06","text":"<ul> <li>Update to iscc-schema 0.3.3</li> <li>Change image normalization instructions</li> <li>Fix issue with exporting cdc cython only functions</li> </ul>"},{"location":"changelog/#021-2022-03-03","title":"[0.2.1] - 2022-03-03","text":"<ul> <li>Cleanup and update dependencies</li> <li>Fix bitarray api change</li> <li>Fix developer commands</li> </ul>"},{"location":"changelog/#020-2022-02-24","title":"[0.2.0] - 2022-02-24","text":"<ul> <li>Complete API refactoring</li> <li>Use Data-URL as input for Meta-Code</li> <li>Use wallet address for ISCC-ID creation</li> <li>Added new Flake-Code (distributed time/random ID)</li> <li>Replaced assertions with exeptions</li> <li>Use secure random functions</li> <li>Retired Python 3.6 support (EOL)</li> <li>Return simple <code>dict</code> objects from generator functions</li> <li>Added ISCC string validation</li> <li>Added multiple helper functions</li> </ul>"},{"location":"changelog/#019-2021-12-17","title":"[0.1.9] - 2021-12-17","text":"<ul> <li>Added warning on non-standard options</li> <li>Added multiformats support</li> <li>Added uri representation</li> <li>Removed redundant cdc_avg_chunk_size option</li> <li>Updated codec format documentation</li> </ul>"},{"location":"changelog/#018-2021-12-12","title":"[0.1.8] - 2021-12-12","text":"<ul> <li>Added conformance tests for all top level functions</li> <li>Added conformance tests to source dir</li> <li>Added conformance module with <code>selftest</code> function</li> <li>Changed gen_image_code to accept normalized pixels instead of stream</li> <li>Changed opts to core_opts</li> <li>Removed image pre-processing and Pillow dependency</li> <li>Fixed readability of conformance tests</li> <li>Fixed soft_hash_video_v0 to accept non-tuple sequences</li> <li>Updated example code</li> </ul>"},{"location":"changelog/#017-2021-12-09","title":"[0.1.7] - 2021-12-09","text":"<ul> <li>Add dotenv for enviroment based configuration</li> <li>Cleanup package toplevel imports</li> <li>Return schema objects for iscc_code and iscc_id</li> <li>Exclude unset and none values from result dicts</li> <li>Add support for multiple code combinations for ISCC-CODE</li> <li>Add support for ISCC-ID based on singular Instance-Code</li> <li>Add initial conformance test system</li> </ul>"},{"location":"changelog/#016-2021-11-29","title":"[0.1.6] - 2021-11-29","text":"<ul> <li>Show counter for ISCC-ID in Code.explain</li> </ul>"},{"location":"changelog/#015-2021-11-28","title":"[0.1.5] - 2021-11-28","text":"<ul> <li>Fix documentation</li> <li>Change metahash creation logic</li> <li>Refactor models</li> <li>Add Content-Code-Mixed</li> <li>Add ISCC-ID</li> <li>Refactor <code>compose</code> to <code>gen_iscc_code</code></li> <li>Refactor <code>models</code> to <code>schema</code></li> </ul>"},{"location":"changelog/#014-2021-11-17","title":"[0.1.4] - 2021-11-17","text":"<ul> <li>Simplified options</li> <li>Optimize video WTA-hash for use with 64-bit granular features</li> </ul>"},{"location":"changelog/#013-2021-11-15","title":"[0.1.3] - 2021-11-15","text":"<ul> <li>Try to compile Cython/C accelerator modules when installing via pip</li> <li>Simplify soft_hash api return values</li> <li>Add .code() method to InstanceHasher, DataHasher</li> <li>Remove granular fingerprint calculation</li> <li>Add more top-level imports</li> </ul>"},{"location":"changelog/#012-2021-11-14","title":"[0.1.2] - 2021-11-14","text":"<ul> <li>Export more functions to toplevel</li> <li>Return schema driven objects from ISCC code generators.</li> </ul>"},{"location":"changelog/#011-2021-11-14","title":"[0.1.1] - 2021-11-14","text":"<ul> <li>Fix packaging problems</li> </ul>"},{"location":"changelog/#010-2021-11-13","title":"[0.1.0] - 2021-11-13","text":"<ul> <li>Initial release</li> </ul>"},{"location":"conformance/","title":"ISCC - Conformance Testing","text":"<p>An application that claims ISCC conformance MUST pass all core functions from the ISCC conformance test suite. The test suite is available as JSON data on GitHub. Test data is structured as follows:</p> <pre><code>{\n\"&lt;function_name&gt;\": {\n\"&lt;test_name&gt;\": {\n\"inputs\": [\"&lt;value1&gt;\", \"&lt;value2&gt;\"],\n\"outputs\": [\"value1&gt;\", \"&lt;value2&gt;\"]\n}\n}\n}\n</code></pre> <p>Inputs that are expected to be <code>raw bytes or byte-streams</code> are embedded as HEX encoded strings in JSON and prefixed with <code>stream:</code> or <code>bytes</code> to support automated decoding during implementation testing.</p> <p>Example</p> <p>Byte-stream outputs in JSON test data: <pre><code>\"gen_data_code_v0\": {\n\"test_0000_two_bytes_64\": {\n\"inputs\": [\n\"stream:ff00\",\n64\n],\n\"outputs\": {\n\"iscc\": \"GAAXL2XYM5BQIAZ3\"\n}\n},\n...\n</code></pre></p>"},{"location":"conformance/#iscc_core.conformance.conformance_testdata","title":"<code>conformance_testdata()</code>","text":"<p>Yield tuples of test data.</p> <p>Returns:</p> Type Description <code>Generator[Tuple[str, Callable, List[Any], List[Any]]]</code> <p>Tuple with testdata (test_name, func_obj, inputs, outputs)</p>"},{"location":"conformance/#iscc_core.conformance.conformance_selftest","title":"<code>conformance_selftest()</code>","text":"<p>Run conformance tests.</p> <p>Returns:</p> Type Description <code>bool</code> <p>whether all tests passed</p>"},{"location":"constants/","title":"ISCC - Types and Constants","text":""},{"location":"constants/#iscc_core.constants.MT","title":"<code>MT</code>","text":""},{"location":"constants/#iscc_core.constants.MT--mt-maintypes","title":"MT - MainTypes","text":"Uint Symbol Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more ISCC-UNITs with common header"},{"location":"constants/#iscc_core.constants.ST","title":"<code>ST</code>","text":""},{"location":"constants/#iscc_core.constants.ST--st-subtypes","title":"ST - SubTypes","text":"Uint Symbol Bits Purpose 0 NONE 0000 For MainTypes that do not specify SubTypes"},{"location":"constants/#iscc_core.constants.ST_CC","title":"<code>ST_CC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_CC--st_cc","title":"ST_CC","text":"<p>SubTypes for <code>MT.CONTENT</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Match on syntactic text similarity 1 IMAGE 0001 Match on perceptual image similarity 2 AUDIO 0010 Match on audio chroma similarity 3 VIDEO 0011 Match on perceptual similarity 4 MIXED 0100 Match on similarity of content codes"},{"location":"constants/#iscc_core.constants.ST_ISCC","title":"<code>ST_ISCC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_ISCC--st_iscc","title":"ST_ISCC","text":"<p>SubTypes for <code>MT.ISCC</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Composite ISCC inlcuding Text-Code 1 IMAGE 0001 Composite ISCC inlcuding Image-Code 2 AUDIO 0010 Composite ISCC inlcuding Audio-Code 3 VIDEO 0011 Composite ISCC inlcuding Video-Code 4 MIXED 0100 Composite ISCC inlcuding Mixed-Code 5 SUM 0101 Composite ISCC inlcuding only Data- and Instance-Code 6 NONE 0110 Composite ISCC including Meta, Data and Instance-Code"},{"location":"constants/#iscc_core.constants.VS","title":"<code>VS</code>","text":""},{"location":"constants/#iscc_core.constants.VS--vs-version","title":"VS - Version","text":"<p>Code Version</p> Uint Symbol Bits Purpose 0 V0 0000 Initial Version of Code without breaking changes"},{"location":"constants/#iscc_core.constants.LN","title":"<code>LN</code>","text":""},{"location":"constants/#iscc_core.constants.LN--ln-length","title":"LN - Length","text":"<p>Valid lengths for hash-digests.</p> <ul> <li>L32 = 32</li> <li>L64 = 64</li> <li>L72 = 72</li> <li>L80 = 80</li> <li>L96 = 96</li> <li>L128 = 128</li> <li>L160 = 160</li> <li>L192 = 192</li> <li>L224 = 224</li> <li>L256 = 256</li> </ul>"},{"location":"constants/#iscc_core.constants.MULTIBASE","title":"<code>MULTIBASE</code>","text":"<p>Supported Multibase encodings.</p> <ul> <li>base16 -&gt; f</li> <li>base32 -&gt; b</li> <li>base32hex -&gt; v</li> <li>base58btc -&gt; z</li> <li>base64url -&gt; u</li> </ul>"},{"location":"iscc_code/","title":"ISCC-CODE","text":"<p>A multi-component identifier for digital media assets.</p> <p>An ISCC-CODE can be generated from the concatenation of the digests of the following five ISCC-UNITs together with a single common header:</p> <ul> <li>Meta-Code - Encodes metadata similarity</li> <li>Semantic-Code - Encodes semantic content similarity (to be developed)</li> <li>Content-Code - Encodes syntactic/perceptual similarity</li> <li>Data-Code - Encodes raw bitstream similarity</li> <li>Instance-Code - Data checksum</li> </ul> <p>The following sequences of ISCC-UNITs are possible:</p> <ul> <li>Data, Instance</li> <li>Content, Data, Instance</li> <li>Semantic, Data, Instance</li> <li>Content, Semantic, Data, Instance</li> <li>Meta, Data, Instance</li> <li>Meta, Content, Data, Instance</li> <li>Meta, Semantic, Data, Instance</li> <li>Meta, Semantic, Content, Data, Instance</li> </ul>"},{"location":"iscc_code/#iscc_core.iscc_code.gen_iscc_code_v0","title":"<code>gen_iscc_code_v0(codes)</code>","text":"<p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iscc_id/","title":"ISCC-ID","text":"<p>A decentralized, owned, and short identifier for digital assets.</p> <p>The ISCC-ID is generated from a similarity-hash of the units of an ISCC-CODE together with a blockchain wallet address. Its SubType designates the blockchain from which the ISCC-ID was minted. The similarity-hash is always at least 64-bits and optionally suffixed with a <code>uvarint</code> endcoded <code>uniqueness counter</code>. The <code>uniqueness counter</code> is added and incremented only if the mint colides with a pre-existing ISCC-ID minted from the same blockchain from a different ISCC-CODE or from an identical ISCC-CODE registered by a different signatory.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id","title":"<code>gen_iscc_id(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate  ISCC-ID from ISCC-CODE with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id_v0","title":"<code>gen_iscc_id_v0(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate an ISCC-ID from an ISCC-CODE with uniqueness counter 'uc' with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.soft_hash_iscc_id_v0","title":"<code>soft_hash_iscc_id_v0(iscc_code, wallet, uc = 0)</code>","text":"<p>Calculate ISCC-ID hash digest from ISCC-CODE with algorithm v0.</p> <p>Accepts an ISCC-CODE or any sequence of ISCC-UNITs.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>ISCC-CODE</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter for ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Digest for ISCC-ID without header but including uniqueness counter.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr","title":"<code>iscc_id_incr(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr_v0","title":"<code>iscc_id_incr_v0(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one (without \"ISCC:\" prefix).</p>"},{"location":"iscc_id/#iscc_core.iscc_id.alg_simhash_from_iscc_id","title":"<code>alg_simhash_from_iscc_id(iscc_id, wallet)</code>","text":"<p>Extract similarity preserving hex-encoded hash digest from ISCC-ID</p> <p>We need to un-xor the ISCC-ID hash digest with the wallet address hash to obtain the similarity preserving bytestring.</p>"},{"location":"iso-reference/","title":"ISCC - ISO Reference","text":"<p>The following functions are the reference implementations of ISO 24138:</p>"},{"location":"iso-reference/#iso-24138-51-meta-code","title":"ISO 24138 / 5.1 Meta-Code","text":"<code>gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits)</code> # <p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"iso-reference/#iso-24138-53-text-code","title":"ISO 24138 / 5.3 Text-Code","text":"<code>gen_text_code_v0(text, bits=ic.core_opts.text_bits)</code> # <p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"iso-reference/#iso-24138-54-image-code","title":"ISO 24138 / 5.4 Image-Code","text":"<code>gen_image_code_v0(pixels, bits=ic.core_opts.image_bits)</code> # <p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p> Source code in <code>iscc_core\\code_content_image.py</code> <pre><code>def gen_image_code_v0(pixels, bits=ic.core_opts.image_bits):\n# type: (Sequence[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Image with algorithm v0.\n    :param Sequence[int] pixels: Normalized image pixels (32x32 flattened gray values)\n    :param int bits: Bit-length of ISCC Content-Code Image (default 64).\n    :return: ISCC object with Content-Code Image.\n    :rtype: ISCC\n    \"\"\"\ndigest = soft_hash_image_v0(pixels, bits=bits)\nimage_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.IMAGE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + image_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-55-audio-code","title":"ISO 24138 / 5.5 Audio-Code","text":"<code>gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits)</code> # <p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p> Source code in <code>iscc_core\\code_content_audio.py</code> <pre><code>def gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits):\n# type: (Iterable[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Audio with algorithm v0.\n    :param Iterable[int] cv: Chromaprint vector\n    :param int bits: Bit-length resulting Content-Code Audio (multiple of 64)\n    :return: ISCC object with Content-Code Audio\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_audio_v0(cv, bits=bits)\naudio_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.AUDIO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + audio_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-56-video-code","title":"ISO 24138 / 5.6 Video-Code","text":"<code>gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits)</code> # <p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p> Source code in <code>iscc_core\\code_content_video.py</code> <pre><code>def gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits):\n# type: (Sequence[ic.FrameSig], int) -&gt; dict\n\"\"\"\n    Create an ISCC Video-Code with algorithm v0.\n    :param ic.FrameSig frame_sigs: Sequence of MP7 frame signatures\n    :param int bits: Bit-length resulting Video-Code (multiple of 64)\n    :return: ISCC object with Video-Code\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_video_v0(frame_sigs, bits=bits)\nvideo_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.VIDEO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + video_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-57-mixed-code","title":"ISO 24138 / 5.7 Mixed-Code","text":"<code>gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits)</code> # <p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p> Source code in <code>iscc_core\\code_content_mixed.py</code> <pre><code>def gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits):\n# type: (Sequence[str], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code-Mixed with algorithm v0.\n    If the provided codes are of mixed length they are stripped to `bits` length for\n    calculation.\n    :param Iterable[str] codes: a list of Content-Codes.\n    :param int bits: Target bit-length of generated Content-Code-Mixed.\n    :return: ISCC object with Content-Code Mixed.\n    :rtype: dict\n    \"\"\"\ndigests = [ic.decode_base32(ic.iscc_clean(code)) for code in codes]\ndigest = soft_hash_codes_v0(digests, bits=bits)\nmixed_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.MIXED,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + mixed_code\nreturn dict(iscc=iscc, parts=list(codes))\n</code></pre>"},{"location":"iso-reference/#iso-24138-58-data-code","title":"ISO 24138 / 5.8 Data-Code","text":"<code>gen_data_code_v0(stream, bits=ic.core_opts.data_bits)</code> # <p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p> Source code in <code>iscc_core\\code_data.py</code> <pre><code>def gen_data_code_v0(stream, bits=ic.core_opts.data_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Data-Code with algorithm v0.\n    :param Stream stream: Input data stream.\n    :param int bits: Bit-length of ISCC Data-Code (default 64).\n    :return: ISCC object with Data-Code\n    :rtype: dict\n    \"\"\"\nhasher = DataHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ndata_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + data_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-59-instance-code","title":"ISO 24138 / 5.9 Instance-Code","text":"<code>gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits)</code> # <p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p> Source code in <code>iscc_core\\code_instance.py</code> <pre><code>def gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Instance-Code with algorithm v0.\n    :param Stream stream: Binary data stream for Instance-Code generation\n    :param int bits: Bit-length of resulting Instance-Code (multiple of 64)\n    :return: ISCC object with Instance-Code and properties: datahash, filesize\n    :rtype: dict\n    \"\"\"\nhasher = InstanceHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ninstance_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + instance_code\ninstance_code_obj = dict(\niscc=iscc,\ndatahash=hasher.multihash(),\nfilesize=hasher.filesize,\n)\nreturn instance_code_obj\n</code></pre>"},{"location":"iso-reference/#iso-24138-60-iscc-code","title":"ISO 24138 / 6.0 ISCC-CODE","text":"<code>gen_iscc_code_v0(codes)</code> # <p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"algorithms/cdc/","title":"ISCC - Content Defined Chunking","text":"<p>Compatible with fastcdc</p>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_chunks","title":"<code>alg_cdc_chunks(data, utf32, avg_chunk_size = ic.core_opts.data_avg_chunk_size)</code>","text":"<p>A generator that yields data-dependent chunks for <code>data</code>.</p> <p>Usage Example:</p> <pre><code>for chunk in cdc_data_chunks(data):\nhash(chunk)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>Raw data for variable sized chunking.</p> required <code>utf32</code> <code>bool</code> <p>If true assume we are chunking text that is utf32 encoded.</p> required <code>avg_chunk_size</code> <code>int</code> <p>Target chunk size in number of bytes.</p> <code>ic.core_opts.data_avg_chunk_size</code> <p>Returns:</p> Type Description <code>Generator[bytes]</code> <p>A generator that yields data chunks of variable sizes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_chunks(data, utf32, avg_chunk_size=ic.core_opts.data_avg_chunk_size):\n# type: (Data, bool, int) -&gt; Generator[bytes, None, None]\n\"\"\"\n    A generator that yields data-dependent chunks for `data`.\n    Usage Example:\n    ```python\n    for chunk in cdc_data_chunks(data):\n        hash(chunk)\n    ```\n    :param bytes data: Raw data for variable sized chunking.\n    :param bool utf32: If true assume we are chunking text that is utf32 encoded.\n    :param int avg_chunk_size: Target chunk size in number of bytes.\n    :return: A generator that yields data chunks of variable sizes.\n    :rtype: Generator[bytes]\n    \"\"\"\nstream = io.BytesIO(data)\nbuffer = stream.read(ic.core_opts.io_read_size)\nif not buffer:\nyield b\"\"\nmi, ma, cs, mask_s, mask_l = alg_cdc_params(avg_chunk_size)\nbuffer = memoryview(buffer)\nwhile buffer:\nif len(buffer) &lt;= ma:\nbuffer = memoryview(bytes(buffer) + stream.read(ic.core_opts.io_read_size))\ncut_point = alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)\n# Make sure cut points are at 4-byte aligned for utf32 encoded text\nif utf32:\ncut_point -= cut_point % 4\nyield bytes(buffer[:cut_point])\nbuffer = buffer[cut_point:]\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_offset","title":"<code>alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)</code>","text":"<p>Find breakpoint offset for a given buffer.</p> <p>Parameters:</p> Name Type Description Default <code>buffer</code> <code>Data</code> <p>The data to be chunked.</p> required <code>mi</code> <code>int</code> <p>Minimum chunk size.</p> required <code>ma</code> <code>int</code> <p>Maximung chunk size.</p> required <code>cs</code> <code>int</code> <p>Center size.</p> required <code>mask_s</code> <code>int</code> <p>Small mask.</p> required <code>mask_l</code> <code>int</code> <p>Large mask.</p> required <p>Returns:</p> Type Description <code>int</code> <p>Offset of dynamic cutpoint in number of bytes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l):\n# type: (ic.Data, int, int, int, int, int) -&gt; int\n\"\"\"\n    Find breakpoint offset for a given buffer.\n    :param Data buffer: The data to be chunked.\n    :param int mi: Minimum chunk size.\n    :param int ma: Maximung chunk size.\n    :param int cs: Center size.\n    :param int mask_s: Small mask.\n    :param int mask_l: Large mask.\n    :return: Offset of dynamic cutpoint in number of bytes.\n    :rtype: int\n    \"\"\"\npattern = 0\ni = mi\nsize = len(buffer)\nbarrier = min(cs, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_s:\nreturn i + 1\ni += 1\nbarrier = min(ma, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_l:\nreturn i + 1\ni += 1\nreturn i\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_params","title":"<code>alg_cdc_params(avg_size: int) -&gt; tuple</code>","text":"<p>Calculate CDC parameters</p> <p>Parameters:</p> Name Type Description Default <code>avg_size</code> <code>int</code> <p>Target average size of chunks in number of bytes.</p> required <p>Returns:</p> Type Description <code>tuple</code> <p>Tuple of (min_size, max_size, center_size, mask_s, mask_l).</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_params(avg_size: int) -&gt; tuple:\n\"\"\"\n    Calculate CDC parameters\n    :param int avg_size: Target average size of chunks in number of bytes.\n    :returns: Tuple of (min_size, max_size, center_size, mask_s, mask_l).\n    \"\"\"\nceil_div = lambda x, y: (x + y - 1) // y\nmask = lambda b: 2**b - 1\nmin_size = avg_size // 4\nmax_size = avg_size * 8\noffset = min_size + ceil_div(min_size, 2)\ncenter_size = avg_size - offset\nbits = round(log2(avg_size))\nmask_s = mask(bits + 1)\nmask_l = mask(bits - 1)\nreturn min_size, max_size, center_size, mask_s, mask_l\n</code></pre>"},{"location":"algorithms/dct/","title":"ISCC - Discrete Cosine Transform","text":""},{"location":"algorithms/dct/#iscc_core.dct.alg_dct","title":"<code>alg_dct(v)</code>","text":"<p>Discrete cosine transform.</p> <p>See: nayuki.io.</p> <p>Parameters:</p> Name Type Description Default <code>v</code> <code>Sequence[float]</code> <p>Input vector for DCT calculation.</p> required <p>Returns:</p> Type Description <code>List</code> <p>DCT Transformed vector.</p> Source code in <code>iscc_core\\dct.py</code> <pre><code>def alg_dct(v):\n# type: (Sequence[float]) -&gt; List\n\"\"\"\n    Discrete cosine transform.\n    See: [nayuki.io](https://www.nayuki.io/page/fast-discrete-cosine-transform-algorithms).\n    :param Sequence[float] v: Input vector for DCT calculation.\n    :return: DCT Transformed vector.\n    :rtype: List\n    \"\"\"\nn = len(v)\nif n == 1:\nreturn list(v)\nelif n == 0 or n % 2 != 0:\nraise ValueError()\nelse:\nhalf = n // 2\nalpha = [(v[i] + v[-(i + 1)]) for i in range(half)]\nbeta = [\n(v[i] - v[-(i + 1)]) / (math.cos((i + 0.5) * math.pi / n) * 2.0) for i in range(half)\n]\nalpha = alg_dct(alpha)\nbeta = alg_dct(beta)\nresult = []\nfor i in range(half - 1):\nresult.append(alpha[i])\nresult.append(beta[i] + beta[i + 1])\nresult.append(alpha[-1])\nresult.append(beta[-1])\nreturn result\n</code></pre>"},{"location":"algorithms/minhash/","title":"ISCC - Minhash","text":""},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash","title":"<code>alg_minhash(features)</code>","text":"<p>Calculate a 64 dimensional minhash integer vector.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>List[int]</code> <p>Minhash vector</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash(features):\n# type: (List[int]) -&gt; List[int]\n\"\"\"\n    Calculate a 64 dimensional minhash integer vector.\n    :param List[int] features: List of integer features\n    :return: Minhash vector\n    :rtype: List[int]\n    \"\"\"\nreturn [\nmin([(((a * f + b) &amp; MAXI64) % MPRIME) &amp; MAXH for f in features]) for a, b in zip(MPA, MPB)\n]\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_64","title":"<code>alg_minhash_64(features)</code>","text":"<p>Create 64-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>64-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_64(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 64-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 64-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 1)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_256","title":"<code>alg_minhash_256(features)</code>","text":"<p>Create 256-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_256(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 256-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 4)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_compress","title":"<code>alg_minhash_compress(mhash, lsb = 4)</code>","text":"<p>Compress minhash vector to byte hash-digest.</p> <p>Concatenates <code>lsb</code> number of least-significant bits from each integer in <code>mhash</code>. For example an <code>mhash</code> with 64 integers and <code>lsb=4</code> will produce a 256-bit summary of the minhash vector.</p> <p>Parameters:</p> Name Type Description Default <code>mhash</code> <code>List[int]</code> <p>List of minhash integer features</p> required <code>lsb</code> <code>int</code> <p>Number of the least significant bits to retain</p> <code>4</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_compress(mhash, lsb=4):\n# type: (List[int], int) -&gt; bytes\n\"\"\"\n    Compress minhash vector to byte hash-digest.\n    Concatenates `lsb` number of least-significant bits from each integer in `mhash`.\n    For example an `mhash` with 64 integers and `lsb=4` will produce a 256-bit summary\n    of the minhash vector.\n    :param List[int] mhash: List of minhash integer features\n    :param int lsb: Number of the least significant bits to retain\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nbits: str = \"\"\nfor bitpos in range(lsb):\nfor h in mhash:\nbits += str(h &gt;&gt; bitpos &amp; 1)\nreturn int(bits, 2).to_bytes((len(bits) + 7) // 8, \"big\")\n</code></pre>"},{"location":"algorithms/simhash/","title":"ISCC - Simhash","text":""},{"location":"algorithms/simhash/#iscc_core.simhash.alg_simhash","title":"<code>alg_simhash(hash_digests)</code>","text":"<p>Creates a similarity preserving hash from a sequence of equal sized hash digests.</p> <p>Parameters:</p> Name Type Description Default <code>hash_digests</code> <code>list</code> <p>A sequence of equaly sized byte-hashes.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity byte-hash</p> Source code in <code>iscc_core\\simhash.py</code> <pre><code>def alg_simhash(hash_digests):\n# type: (list[bytes]) -&gt; bytes\n\"\"\"\n    Creates a similarity preserving hash from a sequence of equal sized hash digests.\n    :param list hash_digests: A sequence of equaly sized byte-hashes.\n    :returns: Similarity byte-hash\n    :rtype: bytes\n    \"\"\"\nn_bytes = len(hash_digests[0])\nn_bits = n_bytes * 8\nvector = [0] * n_bits\nfor digest in hash_digests:\nh = bitarray()\nh.frombytes(digest)\nfor i in range(n_bits):\nvector[i] += h[i]\nminfeatures = len(hash_digests) / 2\nshash = 0\nfor i in range(n_bits):\nif vector[i] &gt;= minfeatures:\nshash |= 1 &lt;&lt; (n_bits - 1 - i)\nreturn shash.to_bytes(n_bytes, \"big\")\n</code></pre>"},{"location":"algorithms/wtahash/","title":"ISCC - Winner Takes All Hash","text":""},{"location":"algorithms/wtahash/#iscc_core.wtahash.alg_wtahash","title":"<code>alg_wtahash(vec: Sequence[float], bits: Sequence[float]) -&gt; bytes</code>","text":"<p>Calculate WTA Hash for vector with 380 values (MP7 frame signature).</p> Source code in <code>iscc_core\\wtahash.py</code> <pre><code>def alg_wtahash(vec: Sequence[float], bits) -&gt; bytes:\n\"\"\"Calculate WTA Hash for vector with 380 values (MP7 frame signature).\"\"\"\nh = []\nfor perm in WTA_VIDEO_ID_PERMUTATIONS:\nv = vec[perm[0]], vec[perm[1]]\nh.append(v.index(max(v)))\nif len(h) == bits:\nbreak\nh = bitarray(h).tobytes()\nreturn h\n</code></pre>"},{"location":"codec/","title":"ISCC - Codec","text":"<p>This module implements encoding, decoding and transcoding functions of ISCC</p>"},{"location":"codec/#codec-overview","title":"Codec Overview","text":""},{"location":"codec/#codec-functions","title":"Codec Functions","text":""},{"location":"codec/#iscc_core.codec.encode_component","title":"<code>encode_component(mtype, stype, version, bit_length, digest)</code>","text":"<p>Encode an ISCC-UNIT inlcuding header and body with standard base32 encoding.</p> <p>Note</p> <p>The <code>length</code> value must be the length in number of bits for the component. If <code>digest</code> has more bits than specified by <code>length</code> it wil be truncated.</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>Maintype of unit (0-6)</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit depending on MainType (0-5)</p> required <code>version</code> <code>Version</code> <p>Version of unit algorithm (0).</p> required <code>bit_length</code> <code>length</code> <p>Length of unit, in number of bits (multiple of 32)</p> required <code>digest</code> <code>bytes</code> <p>The hash digest of the unit.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32 encoded ISCC-UNIT.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_component(mtype, stype, version, bit_length, digest):\n# type: (MainType, SubType, Version, Length, bytes) -&gt; str\n\"\"\"\n    Encode an ISCC-UNIT inlcuding header and body with standard base32 encoding.\n    !!! note\n        The `length` value must be the **length in number of bits** for the component.\n        If `digest` has more bits than specified by `length` it wil be truncated.\n    :param MainType mtype: Maintype of unit (0-6)\n    :param SubType stype: SubType of unit depending on MainType (0-5)\n    :param Version version: Version of unit algorithm (0).\n    :param length bit_length: Length of unit, in number of bits (multiple of 32)\n    :param bytes digest: The hash digest of the unit.\n    :return: Base32 encoded ISCC-UNIT.\n    :rtype: str\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.ID, MT.FLAKE):\nencoded_length = encode_length(mtype, bit_length)\nelif mtype == MT.ISCC:\nraise ValueError(f\"{mtype} is not a unit\")\nelse:\nraise ValueError(f\"Illegal MainType {mtype}\")\nnbytes = bit_length // 8\nheader = encode_header(mtype, stype, version, encoded_length)\nbody = digest[:nbytes]\ncomponent_code = encode_base32(header + body)\nreturn component_code\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_header","title":"<code>encode_header(mtype, stype, version = 0, length = 1)</code>","text":"<p>Encodes header values with nibble-sized (4-bit) variable-length encoding. The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles is uneven it is padded with 4-bit <code>0000</code> at the end.</p> <p>Warning</p> <p>The length value must be encoded beforhand because its semantics depend on the MainType (see <code>encode_length</code> function).</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>MainType of unit.</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit.</p> required <code>version</code> <code>Version</code> <p>Version of component algorithm.</p> <code>0</code> <code>length</code> <code>Length</code> <p>length value of unit (1 means 64-bits for standard units)</p> <code>1</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Varnibble stream encoded ISCC header as bytes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_header(mtype, stype, version=0, length=1):\n# type: (MainType, SubType, Version, Length) -&gt; bytes\n\"\"\"\n    Encodes header values with nibble-sized (4-bit) variable-length encoding.\n    The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles\n    is uneven it is padded with 4-bit `0000` at the end.\n    !!! warning\n        The length value must be encoded beforhand because its semantics depend on\n        the MainType (see `encode_length` function).\n    :param MainType mtype: MainType of unit.\n    :param SubType stype: SubType of unit.\n    :param Version version: Version of component algorithm.\n    :param Length length: length value of unit (1 means 64-bits for standard units)\n    :return: Varnibble stream encoded ISCC header as bytes.\n    :rtype: bytes\n    \"\"\"\nheader = bitarray()\nfor n in (mtype, stype, version, length):\nheader += encode_varnibble(n)\n# Append zero-padding if required (right side, least-significant bits).\nheader.fill()\nreturn header.tobytes()\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_header","title":"<code>decode_header(data)</code>","text":"<p>Decodes varnibble encoded header and returns it together with <code>tail data</code>.</p> <p>Tail data is included to enable decoding of sequential ISCCs. The returned tail data must be truncated to decode_length(r[0], r[3]) bits to recover the actual hash-bytes.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>ISCC bytes</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>(MainType, SubType, Version, length, TailData)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_header(data):\n# type: (bytes) -&gt; IsccTuple\n\"\"\"\n    Decodes varnibble encoded header and returns it together with `tail data`.\n    Tail data is included to enable decoding of sequential ISCCs. The returned tail\n    data must be truncated to decode_length(r[0], r[3]) bits to recover the actual\n    hash-bytes.\n    :param bytes data: ISCC bytes\n    :return: (MainType, SubType, Version, length, TailData)\n    :rtype: IsccTuple\n    \"\"\"\nresult = []\nba = bitarray()\nba.frombytes(data)\ndata = ba\nfor _ in range(4):\nvalue, data = decode_varnibble(data)\nresult.append(value)\n# Strip 4-bit padding if required\nif len(data) % 8 and data[:4] == bitarray(\"0000\"):\ndata = data[4:]\nresult.append(data.tobytes())\nreturn tuple(result)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_varnibble","title":"<code>encode_varnibble(n)</code>","text":"<p>Writes integer to variable length sequence of 4-bit chunks.</p> <p>Variable-length encoding scheme:</p> prefix bits nibbles data bits unsigned range 0 1 3 0 - 7 10 2 6 8 - 71 110 3 9 72 - 583 1110 4 12 584 - 4679 <p>Parameters:</p> Name Type Description Default <code>n</code> <code>int</code> <p>Positive integer to be encoded as varnibble (0-4679)</p> required <p>Returns:</p> Type Description <code>bitarray</code> <p>Varnibble encoded integera</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_varnibble(n):\n# type: (int) -&gt; bitarray\n\"\"\"\n    Writes integer to variable length sequence of 4-bit chunks.\n    Variable-length encoding scheme:\n    ------------------------------------------------------\n    | prefix bits | nibbles | data bits | unsigned range |\n    | ----------- | ------- | --------- | -------------- |\n    | 0           | 1       | 3         | 0 - 7          |\n    | 10          | 2       | 6         | 8 - 71         |\n    | 110         | 3       | 9         | 72 - 583       |\n    | 1110        | 4       | 12        | 584 - 4679     |\n    :param int n: Positive integer to be encoded as varnibble (0-4679)\n    :return: Varnibble encoded integera\n    :rtype: bitarray\n    \"\"\"\nif 0 &lt;= n &lt; 8:\nreturn int2ba(n, length=4)\nelif 8 &lt;= n &lt; 72:\nreturn bitarray(\"10\") + int2ba(n - 8, length=6)\nelif 72 &lt;= n &lt; 584:\nreturn bitarray(\"110\") + int2ba(n - 72, length=9)\nelif 584 &lt;= n &lt; 4680:\nreturn bitarray(\"1110\") + int2ba(n - 584, length=12)\nelse:\nraise ValueError(\"Value must be between 0 and 4679\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_varnibble","title":"<code>decode_varnibble(b)</code>","text":"<p>Reads first varnibble, returns its integer value and remaining bits.</p> <p>Parameters:</p> Name Type Description Default <code>b</code> <code>bitarray</code> <p>Array of header bits</p> required <p>Returns:</p> Type Description <code>Tuple[int, bitarray]</code> <p>A tuple of the integer value of first varnible and the remaining bits.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_varnibble(b):\n# type: (bitarray) -&gt; Tuple[int, bitarray]\n\"\"\"Reads first varnibble, returns its integer value and remaining bits.\n    :param bitarray b: Array of header bits\n    :return: A tuple of the integer value of first varnible and the remaining bits.\n    :rtype: Tuple[int, bitarray]\n    \"\"\"\nbits = len(b)\nif bits &gt;= 4 and b[0] == 0:\nreturn ba2int(b[:4]), b[4:]\nif bits &gt;= 8 and b[1] == 0:\nreturn ba2int(b[2:8]) + 8, b[8:]\nif bits &gt;= 12 and b[2] == 0:\nreturn ba2int(b[3:12]) + 72, b[12:]\nif bits &gt;= 16 and b[3] == 0:\nreturn ba2int(b[4:16]) + 584, b[16:]\nraise ValueError(\"Invalid bitarray\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_units","title":"<code>encode_units(units)</code>","text":"<p>Encodes a combination of ISCC units to an integer between 0-7 to be used as length value for the final encoding of MT.ISCC</p> <p>Parameters:</p> Name Type Description Default <code>units</code> <code>Tuple</code> <p>A tuple of a MainType combination (can be empty)</p> required <p>Returns:</p> Type Description <code>int</code> <p>Integer value to be used as length-value for header encoding</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_units(units):\n# type: (Tuple[MT, ...]) -&gt; int\n\"\"\"\n    Encodes a combination of ISCC units to an integer between 0-7 to be used as length\n    value for the final encoding of MT.ISCC\n    :param Tuple units: A tuple of a MainType combination (can be empty)\n    :return: Integer value to be used as length-value for header encoding\n    :rtype: int\n    \"\"\"\nreturn UNITS.index(units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_units","title":"<code>decode_units(unit_id)</code>","text":"<p>Decodes an ISCC header length value that has been encoded with a unit_id to an ordered tuple of MainTypes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_units(unit_id):\n# type: (int) -&gt; Tuple[MT, ...]\n\"\"\"\n    Decodes an ISCC header length value that has been encoded with a unit_id to an\n    ordered tuple of MainTypes.\n    \"\"\"\nunits = sorted(UNITS[unit_id])\nreturn tuple(MT(u) for u in units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_length","title":"<code>encode_length(mtype, length)</code>","text":"<p>Encode length to integer value for header encoding.</p> <p>The <code>length</code> value has MainType-specific semantics:</p> <p>For MainTypes <code>META</code>, <code>SEMANTIC</code>, <code>CONTENT</code>, <code>DATA</code>, <code>INSTANCE</code>:</p> <pre><code>Length means number of bits for the body.\nLength is encoded as the multiple of 32-bit chunks (0 being 32bits)\nExamples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n</code></pre> <p>For MainType <code>ISCC</code>:</p> <pre><code>MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\noptional. Length means the composition of optional 64-bit units included\nin the ISCC composite.\n\nExamples:\n    No optional units      -&gt; 0000 -&gt; 0\n    CONTENT                -&gt; 0001 -&gt; 1\n    SEMANTIC               -&gt; 0010 -&gt; 2\n    SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n    META                   -&gt; 0100 -&gt; 4\n    META, CONTENT          -&gt; 0101 -&gt; 5\n    ...\n</code></pre> <p>For MainType <code>ID</code>:</p> <pre><code>Lengths means number the number of bits for the body including the counter\nLength is encoded as number of bytes of the counter (64-bit body is implicit)\nExamples:\n    64 -&gt; 0 (No counter)\n    72 -&gt; 1 (One byte counter)\n    80 -&gt; 2 (Two byte counter)\n    ...\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>The MainType for which to encode the length value.</p> required <code>length</code> <code>Length</code> <p>The length expressed according to the semantics of the type</p> required <p>Returns:</p> Type Description <code>int</code> <p>The length value encoded as integer for use with write_header.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_length(mtype, length):\n# type: (MainType, Length) -&gt; int\n\"\"\"\n    Encode length to integer value for header encoding.\n    The `length` value has MainType-specific semantics:\n    For MainTypes `META`, `SEMANTIC`, `CONTENT`, `DATA`, `INSTANCE`:\n        Length means number of bits for the body.\n        Length is encoded as the multiple of 32-bit chunks (0 being 32bits)\n        Examples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n    For MainType `ISCC`:\n        MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\n        optional. Length means the composition of optional 64-bit units included\n        in the ISCC composite.\n        Examples:\n            No optional units      -&gt; 0000 -&gt; 0\n            CONTENT                -&gt; 0001 -&gt; 1\n            SEMANTIC               -&gt; 0010 -&gt; 2\n            SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n            META                   -&gt; 0100 -&gt; 4\n            META, CONTENT          -&gt; 0101 -&gt; 5\n            ...\n    For MainType `ID`:\n        Lengths means number the number of bits for the body including the counter\n        Length is encoded as number of bytes of the counter (64-bit body is implicit)\n        Examples:\n            64 -&gt; 0 (No counter)\n            72 -&gt; 1 (One byte counter)\n            80 -&gt; 2 (Two byte counter)\n            ...\n    :param MainType mtype: The MainType for which to encode the length value.\n    :param Length length: The length expressed according to the semantics of the type\n    :return: The length value encoded as integer for use with write_header.\n    :rtype: int\n    \"\"\"\nerror = f\"Invalid length {length} for MainType {mtype}\"\n# standard case (length field denotes number of 32-bit chunks, 0 being 32-bits)\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nif length &gt;= 32 and not length % 32:\nreturn (length // 32) - 1\nraise ValueError(error)\n# flag type encoding of included components (pass through as encoded out-of-band)\nelif mtype == MT.ISCC:\nif 0 &lt;= length &lt;= 7:\nreturn length\nraise ValueError(error)\n# counter byte length encoding\nelif mtype == MT.ID:\nif 64 &lt;= length &lt;= 96:\nreturn (length - 64) // 8\nraise ValueError(error)\nelse:\nraise ValueError(error)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_length","title":"<code>decode_length(mtype, length)</code>","text":"<p>Dedoce raw length value from ISCC header to length of digest in number of bits.</p> <p>Decodes a raw header integer value in to its semantically meaningfull value (e.g. number of bits)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_length(mtype, length):\n# type: (MainType, Length) -&gt; LN\n\"\"\"\n    Dedoce raw length value from ISCC header to length of digest in number of bits.\n    Decodes a raw header integer value in to its semantically meaningfull value (e.g.\n    number of bits)\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nreturn LN((length + 1) * 32)\nelif mtype == MT.ISCC:\nreturn LN(len(decode_units(length)) * 64 + 128)\nelif mtype == MT.ID:\nreturn LN(length * 8 + 64)\nelse:\nraise ValueError(f\"Invalid length {length} for MainType {mtype}\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32","title":"<code>encode_base32(data)</code>","text":"<p>Standard RFC4648 base32 encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base32 encoding without padding.\n    \"\"\"\nreturn b32encode(data).decode(\"ascii\").rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32","title":"<code>decode_base32(code)</code>","text":"<p>Standard RFC4648 base32 decoding without padding and with casefolding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base32 decoding without padding and with casefolding.\n    \"\"\"\n# python stdlib does not support base32 without padding, so we have to re-pad.\ncl = len(code)\npad_length = math.ceil(cl / 8) * 8 - cl\nreturn bytes(b32decode(code + \"=\" * pad_length, casefold=True))\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_decompose","title":"<code>iscc_decompose(iscc_code)</code>","text":"<p>Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.</p> <p>A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated by a hyphen.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decompose(iscc_code):\n# type: (str) -&gt; List[str]\n\"\"\"\n    Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.\n    A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated\n    by a hyphen.\n    \"\"\"\niscc_code = iscc_clean(iscc_code)\ncomponents = []\nraw_code = decode_base32(iscc_code)\nwhile raw_code:\nmt, st, vs, ln, body = decode_header(raw_code)\n# standard ISCC-UNIT with tail continuation\nif mt != MT.ISCC:\nln_bits = decode_length(mt, ln)\ncode = encode_component(mt, st, vs, ln_bits, body[: ln_bits // 8])\ncomponents.append(code)\nraw_code = body[ln_bits // 8 :]\ncontinue\n# ISCC-CODE\nmain_types = decode_units(ln)\n# rebuild dynamic units (META, SEMANTIC, CONTENT)\nfor idx, mtype in enumerate(main_types):\nstype = ST.NONE if mtype == MT.META else st\ncode = encode_component(mtype, stype, vs, 64, body[idx * 8 :])\ncomponents.append(code)\n# rebuild static units (DATA, INSTANCE)\ndata_code = encode_component(MT.DATA, ST.NONE, vs, 64, body[-16:-8])\ninstance_code = encode_component(MT.INSTANCE, ST.NONE, vs, 64, body[-8:])\ncomponents.extend([data_code, instance_code])\nbreak\nreturn components\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_normalize","title":"<code>iscc_normalize(iscc_code)</code>","text":"<p>Normalize an ISCC to its canonical form.</p> <p>The canonical form of an ISCC is its shortest base32 encoded representation prefixed with the string <code>ISCC:</code>.</p> <p>Possible valid inputs:</p> <pre><code>MEACB7X7777574L6\nISCC:MEACB7X7777574L6\nfcc010001657fe7cafe9791bb\niscc:maagztfqttvizpjr\nIscc:Maagztfqttvizpjr\n</code></pre> <p>Info</p> <p>A concatenated sequence of codes will be composed into a single ISCC of MainType <code>MT.ISCC</code> if possible.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core\n&gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Normalized ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_normalize(iscc_code):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize an ISCC to its canonical form.\n    The canonical form of an ISCC is its shortest base32 encoded representation\n    prefixed with the string `ISCC:`.\n    Possible valid inputs:\n        MEACB7X7777574L6\n        ISCC:MEACB7X7777574L6\n        fcc010001657fe7cafe9791bb\n        iscc:maagztfqttvizpjr\n        Iscc:Maagztfqttvizpjr\n    !!! info\n        A concatenated sequence of codes will be composed into a single ISCC of MainType\n        `MT.ISCC` if possible.\n    !!! example\n        ``` py\n        &gt;&gt;&gt; import iscc_core\n        &gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n        'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n        ```\n    :param str iscc_code: Any valid ISCC string\n    :return: Normalized ISCC\n    :rtype: str\n    \"\"\"\nfrom iscc_core.iscc_code import gen_iscc_code_v0\ndecoders = {\nMULTIBASE.base16.value: bytes.fromhex,  # f\nMULTIBASE.base32.value: decode_base32,  # b\nMULTIBASE.base32hex.value: decode_base32hex,  # v\nMULTIBASE.base58btc.value: base58.b58decode,  # z\nMULTIBASE.base64url.value: decode_base64,  # u\n}\n# Transcode to base32 if &lt;multibase&gt;&lt;multicodec&gt; encoded\nmultibase_prefix = iscc_code[0]\nif multibase_prefix in decoders.keys():\ndecoder = decoders[multibase_prefix]\ndecoded = decoder(iscc_code[1:])\nif not decoded.startswith(MC_PREFIX):\nraise ValueError(f\"Malformed multiformat codec: {decoded[:2]}\")\niscc_code = encode_base32(decoded[2:])\nelse:\nprefix = iscc_code.upper().replace(\"ISCC:\", \"\")[:2]\nif prefix not in PREFIXES:\nraise ValueError(f\"ISCC starts with invalid prefix {prefix}\")\ndecomposed = iscc_decompose(iscc_code)\nrecomposed = gen_iscc_code_v0(decomposed)[\"iscc\"] if len(decomposed) &gt;= 2 else decomposed[0]\nreturn f\"ISCC:{recomposed}\" if not recomposed.startswith(\"ISCC:\") else recomposed\n</code></pre>"},{"location":"codec/#alternate-encodings","title":"Alternate Encodings","text":""},{"location":"codec/#iscc_core.codec.encode_base64","title":"<code>encode_base64(data)</code>","text":"<p>Standard RFC4648 base64url encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base64(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base64url encoding without padding.\n    \"\"\"\ncode = urlsafe_b64encode(data).decode(\"ascii\")\nreturn code.rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base64","title":"<code>decode_base64(code)</code>","text":"<p>Standard RFC4648 base64url decoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base64(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base64url decoding without padding.\n    \"\"\"\npadding = 4 - (len(code) % 4)\nstring = code + (\"=\" * padding)\nreturn urlsafe_b64decode(string)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32hex","title":"<code>encode_base32hex(data)</code>","text":"<p>RFC4648 Base32hex encoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32hex(data):\n# type: (bytes) -&gt;  str\n\"\"\"\n    RFC4648 Base32hex encoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = encode_base32(data)\nreturn b32.translate(b32_to_hex)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32hex","title":"<code>decode_base32hex(code)</code>","text":"<p>RFC4648 Base32hex decoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32hex(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    RFC4648 Base32hex decoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = code.translate(hex_to_b32)\nreturn decode_base32(b32)\n</code></pre>"},{"location":"codec/#helper-functions","title":"Helper Functions","text":""},{"location":"codec/#iscc_core.codec.iscc_decode","title":"<code>iscc_decode(iscc)</code>","text":"<p>Decode ISCC to an IsccTuple</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>ISCC decoded to a tuple</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decode(iscc):\n# type: (str) -&gt; IsccTuple\n\"\"\"\n    Decode ISCC to an IsccTuple\n    :param str iscc: ISCC string\n    :return: ISCC decoded to a tuple\n    :rtype: IsccTuple\n    \"\"\"\niscc = iscc_clean(iscc_normalize(iscc))\ndata = decode_base32(iscc)\nreturn decode_header(data)\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_explain","title":"<code>iscc_explain(iscc)</code>","text":"<p>Convert ISCC to a human-readable representation</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Human-readable representation of ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_explain(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Convert ISCC to a human-readable representation\n    :param str iscc: ISCC string\n    :return: Human-readable representation of ISCC\n    :rtype: str\n    \"\"\"\ntid = iscc_type_id(iscc)\nfields = iscc_decode(iscc)\nif fields[0] == MT.ID:\ncounter_bytes = fields[-1][8:]\nif counter_bytes:\ncounter = uvarint.decode(counter_bytes)\nhex_hash = fields[-1][:8].hex()\nreturn f\"{tid}-{hex_hash}-{counter.integer}\"\nhex_hash = fields[-1].hex()\nreturn f\"{tid}-{hex_hash}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_type_id","title":"<code>iscc_type_id(iscc)</code>","text":"<p>Extract and convert ISCC HEADER to a readable Type-ID string.</p> <p>Type-ids can be used as names in databases to index ISCC-UNITs seperatly.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Unique Type-ID string</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_type_id(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Extract and convert ISCC HEADER to a readable Type-ID string.\n    Type-ids can be used as names in databases to index ISCC-UNITs seperatly.\n    :param str iscc: ISCC string\n    :return: Unique Type-ID string\n    :rtype: str\n    \"\"\"\nfields = iscc_decode(iscc)\nmtype = MT(fields[0])\nstype = SUBTYPE_MAP[fields[0]](fields[1])\nif mtype == MT.ISCC:\nmtypes = decode_units(fields[3])\nlength = \"\".join([t.name[0] for t in mtypes]) + \"DI\"\nelse:\nlength = decode_length(fields[0], fields[3])\nversion = VS(fields[2])\nreturn f\"{mtype.name}-{stype.name}-{version.name}-{length}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_validate","title":"<code>iscc_validate(iscc, strict = True)</code>","text":"<p>Validate that a given string is a strictly well-formed ISCC.</p> <p>A strictly well-formed ISCC is:</p> <ul> <li>an ISCC-CODE or ISCC-UNIT</li> <li>encoded with base32 upper without padding</li> <li>has a valid combination of header values</li> <li>is represented in its canonical form</li> </ul> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <code>strict</code> <code>bool</code> <p>Raise an exeption if validation fails (default True)</p> <code>True</code> <p>Returns:</p> Type Description <code>bool</code> <p>True if sting is valid else false. (raises ValueError in strict mode)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_validate(iscc, strict=True):\n# type: (str, bool) -&gt; bool\n\"\"\"\n    Validate that a given string is a *strictly well-formed* ISCC.\n    A *strictly well-formed* ISCC is:\n    - an ISCC-CODE or ISCC-UNIT\n    - encoded with base32 upper without padding\n    - has a valid combination of header values\n    - is represented in its canonical form\n    :param str iscc: ISCC string\n    :param bool strict: Raise an exeption if validation fails (default True)\n    :return: True if sting is valid else false. (raises ValueError in strict mode)\n    :rtype: bool\n    \"\"\"\n# Basic regex validation\nmatch = re.match(\"^ISCC:[A-Z2-7]{10,60}$\", iscc)\nif not match:\nif strict:\nraise ValueError(\"ISCC string does not match ^ISCC:[A-Z2-7]{10,60}$\")\nelse:\nreturn False\n# Base32 encoding test\ntry:\ndecode_base32(iscc.split(\":\")[1])\nexcept Exception as e:\nif strict:\nraise ValueError(e)\nelse:\nreturn False\ncleaned = iscc_clean(iscc)\n# Prefix test\nprefix = cleaned[:2]\nif prefix not in PREFIXES:\nif strict:\nraise ValueError(f\"Header starts with invalid sequence {prefix}\")\nelse:\nreturn False\n# Version test\nm, s, v, l, t = decode_header(decode_base32(cleaned))\nif v != 0:\nif strict:\nraise ValueError(f\"Unknown version {v} in version header\")\nelse:\nreturn False\n# Length test\nexpected_nbyptes = decode_length(m, l).value // 8\nactual_nbyptes = len(t)\nif expected_nbyptes != actual_nbyptes:\nif strict:\nraise ValueError(f\"Header expects {expected_nbyptes} but got {actual_nbyptes} bytes\")\nelse:\nreturn False\nreturn True\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_clean","title":"<code>iscc_clean(iscc)</code>","text":"<p>Cleanup ISCC string.</p> <p>Removes leading scheme, dashes, leading/trailing whitespace.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Cleaned ISCC string.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_clean(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Cleanup ISCC string.\n    Removes leading scheme, dashes, leading/trailing whitespace.\n    :param str iscc: Any valid ISCC string\n    :return: Cleaned ISCC string.\n    :rtype: str\n    \"\"\"\nsplit = [part.strip() for part in iscc.strip().split(\":\")]\nif len(split) == 1:\ncode = split[0]\n# remove dashes if not multiformat\nif code[0] not in list(MULTIBASE):\ncode = code.replace(\"-\", \"\")\nreturn code\nelif len(split) == 2:\nscheme, code = split\nif scheme.lower() != \"iscc\":\nraise ValueError(f\"Invalid scheme: {scheme}\")\nreturn code.replace(\"-\", \"\")\nelse:\nraise ValueError(f\"Malformed ISCC string: {iscc}\")\n</code></pre>"},{"location":"options/options/","title":"ISCC-CORE - Configuration Options","text":"<p>Options for the iscc-core package can be configured using environment variables. Variables are loaded as class-attributes on the <code>CoreOptions</code> instance. Environment variables are named like the class-attribute but prefixed with <code>ISCC_CORE_</code> and upper-cased.</p> <p>Example how to access configuration options</p> <pre><code>import iscc_core as ic\n# To access ISCC_CORE_TEXT_NGRAM_SIZE setting use\ntext_ngram_size: int = ic.core_opts.text_ngram_size\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions","title":"CoreOptions","text":"<p>Parameters with defaults for ISCC calculations.</p>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_bits","title":"meta_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_bits: int = Field(\n64,\ndescription=\"Default length of generated Meta-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_name","title":"meta_trim_name  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_trim_name: int = Field(\n128, description=\"Trim `name` to this mumber of bytes\"\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_description","title":"meta_trim_description  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_trim_description: int = Field(\n4096,\ndescription=\"Trim `description` to this number of bytes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_text","title":"meta_ngram_size_text  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_ngram_size_text: int = Field(\n3,\ndescription=\"Sliding window width (characters) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_bytes","title":"meta_ngram_size_bytes  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_ngram_size_bytes: int = Field(\n4,\ndescription=\"Sliding window width (bytes) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_bits","title":"text_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Text in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_ngram_size","title":"text_ngram_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_ngram_size: int = Field(\n13,\ndescription=\"Number of characters per feature hash (size of sliding window)\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_unicode_filter","title":"text_unicode_filter  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_unicode_filter: frozenset = Field(\nfrozenset({\"C\", \"M\", \"P\"}),\ndescription=\"Unicode categories to remove during text normalization\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_newlines","title":"text_newlines  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_newlines: frozenset = Field(\nfrozenset(\n{\n\"\\n\",\n\"\\x0b\",\n\"\\x0c\",\n\"\\r\",\n\"\\x85\",\n\"\\u2028\",\n\"\\u2029\",\n}\n),\ndescription=\"Characters regarded as newline characters for normalization purposes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.image_bits","title":"image_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>image_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Image in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.audio_bits","title":"audio_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>audio_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Audio in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.video_bits","title":"video_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>video_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Video in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_bits","title":"data_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>data_bits: int = Field(\n64,\ndescription=\"Default length of generated Data-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_avg_chunk_size","title":"data_avg_chunk_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>data_avg_chunk_size: int = Field(\n1024,\ndescription=\"Target chunk size for data chunking in number of bytes.\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.instance_bits","title":"instance_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>instance_bits: int = Field(\n64,\ndescription=\"Default length of generated Instance-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.mixed_bits","title":"mixed_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>mixed_bits: int = Field(\n64,\ndescription=\"Default length of generated Mixed-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.io_read_size","title":"io_read_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>io_read_size: int = Field(\n2097152,\ndescription=\"File read buffer size in bytes for hashing operations\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.cdc_gear","title":"cdc_gear  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>cdc_gear: Tuple = Field(\n(\n1553318008,\n574654857,\n759734804,\n310648967,\n1393527547,\n1195718329,\n694400241,\n1154184075,\n1319583805,\n1298164590,\n122602963,\n989043992,\n1918895050,\n933636724,\n1369634190,\n1963341198,\n1565176104,\n1296753019,\n1105746212,\n1191982839,\n1195494369,\n29065008,\n1635524067,\n722221599,\n1355059059,\n564669751,\n1620421856,\n1100048288,\n1018120624,\n1087284781,\n1723604070,\n1415454125,\n737834957,\n1854265892,\n1605418437,\n1697446953,\n973791659,\n674750707,\n1669838606,\n320299026,\n1130545851,\n1725494449,\n939321396,\n748475270,\n554975894,\n1651665064,\n1695413559,\n671470969,\n992078781,\n1935142196,\n1062778243,\n1901125066,\n1935811166,\n1644847216,\n744420649,\n2068980838,\n1988851904,\n1263854878,\n1979320293,\n111370182,\n817303588,\n478553825,\n694867320,\n685227566,\n345022554,\n2095989693,\n1770739427,\n165413158,\n1322704750,\n46251975,\n710520147,\n700507188,\n2104251000,\n1350123687,\n1593227923,\n1756802846,\n1179873910,\n1629210470,\n358373501,\n807118919,\n751426983,\n172199468,\n174707988,\n1951167187,\n1328704411,\n2129871494,\n1242495143,\n1793093310,\n1721521010,\n306195915,\n1609230749,\n1992815783,\n1790818204,\n234528824,\n551692332,\n1930351755,\n110996527,\n378457918,\n638641695,\n743517326,\n368806918,\n1583529078,\n1767199029,\n182158924,\n1114175764,\n882553770,\n552467890,\n1366456705,\n934589400,\n1574008098,\n1798094820,\n1548210079,\n821697741,\n601807702,\n332526858,\n1693310695,\n136360183,\n1189114632,\n506273277,\n397438002,\n620771032,\n676183860,\n1747529440,\n909035644,\n142389739,\n1991534368,\n272707803,\n1905681287,\n1210958911,\n596176677,\n1380009185,\n1153270606,\n1150188963,\n1067903737,\n1020928348,\n978324723,\n962376754,\n1368724127,\n1133797255,\n1367747748,\n1458212849,\n537933020,\n1295159285,\n2104731913,\n1647629177,\n1691336604,\n922114202,\n170715530,\n1608833393,\n62657989,\n1140989235,\n381784875,\n928003604,\n449509021,\n1057208185,\n1239816707,\n525522922,\n476962140,\n102897870,\n132620570,\n419788154,\n2095057491,\n1240747817,\n1271689397,\n973007445,\n1380110056,\n1021668229,\n12064370,\n1186917580,\n1017163094,\n597085928,\n2018803520,\n1795688603,\n1722115921,\n2015264326,\n506263638,\n1002517905,\n1229603330,\n1376031959,\n763839898,\n1970623926,\n1109937345,\n524780807,\n1976131071,\n905940439,\n1313298413,\n772929676,\n1578848328,\n1108240025,\n577439381,\n1293318580,\n1512203375,\n371003697,\n308046041,\n320070446,\n1252546340,\n568098497,\n1341794814,\n1922466690,\n480833267,\n1060838440,\n969079660,\n1836468543,\n2049091118,\n2023431210,\n383830867,\n2112679659,\n231203270,\n1551220541,\n1377927987,\n275637462,\n2110145570,\n1700335604,\n738389040,\n1688841319,\n1506456297,\n1243730675,\n258043479,\n599084776,\n41093802,\n792486733,\n1897397356,\n28077829,\n1520357900,\n361516586,\n1119263216,\n209458355,\n45979201,\n363681532,\n477245280,\n2107748241,\n601938891,\n244572459,\n1689418013,\n1141711990,\n1485744349,\n1181066840,\n1950794776,\n410494836,\n1445347454,\n2137242950,\n852679640,\n1014566730,\n1999335993,\n1871390758,\n1736439305,\n231222289,\n603972436,\n783045542,\n370384393,\n184356284,\n709706295,\n1453549767,\n591603172,\n768512391,\n854125182,\n),\ndescription=\"Random gear vector\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformanc_critical","title":"conformanc_critical  <code>module-attribute</code>","text":"<pre><code>conformanc_critical = {\n\"meta_trim_name\",\n\"meta_trim_description\",\n\"meta_ngram_size_text\",\n\"meta_ngram_size_bytes\",\n\"text_ngram_size\",\n\"text_unicode_filter\",\n\"text_newlines\",\n\"data_avg_chunk_size\",\n\"cdc_gear\",\n}\n</code></pre>"},{"location":"options/options/#iscc_core.options.has_logged_confromance","title":"has_logged_confromance  <code>module-attribute</code>","text":"<pre><code>has_logged_confromance = False\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformance_check_options","title":"conformance_check_options","text":"<pre><code>conformance_check_options(opts)\n</code></pre> <p>Check and log if options have non-default conformance critical values</p>"},{"location":"options/options/#iscc_core.options.core_opts","title":"core_opts  <code>module-attribute</code>","text":"<pre><code>core_opts = CoreOptions()\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformant_options","title":"conformant_options  <code>module-attribute</code>","text":"<pre><code>conformant_options = conformance_check_options(core_opts)\n</code></pre>"},{"location":"units/","title":"ISCC - UNITs","text":"<p>A standard ISCC-CODE is build from multiple ISCC-UNITs. Each unit serve a different purpose.</p>"},{"location":"units/code_data/","title":"ISCC - Data-Code","text":"<p>A similarity perserving hash for binary data (soft hash).</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code","title":"<code>gen_data_code(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create a similarity preserving ISCC Data-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code_v0","title":"<code>gen_data_code_v0(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.soft_hash_data_v0","title":"<code>soft_hash_data_v0(stream)</code>","text":"<p>Create a similarity preserving Data-Hash digest</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Data-Hash (soft-hash) digest used as body for Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0","title":"<code>DataHasherV0</code>","text":"<p>Incremental Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.__init__","title":"<code>__init__(data = None)</code>","text":"<p>Create a DataHasher</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Optional[Data]</code> <p>initial payload for hashing.</p> <code>None</code>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.digest","title":"<code>digest()</code>","text":"<p>Calculate 256-bit minhash digest from feature hashes.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.code","title":"<code>code(bits = ic.core_opts.data_bits)</code>","text":"<p>Encode digest as an ISCC Data-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Data-Code</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Data-Code</p>"},{"location":"units/code_flake/","title":"ISCC - Flake-Code","text":"<p>A unique, time-sorted identifier composed of an 48-bit timestamp and 16 to 208 bit randomness.</p> <p>The ISCC Flake-Code is a unique identifier for distributed ID generation. The 64-bit version can be used as efficient surrogate key in database systems. It has guaranteed uniqueness if generated from a singele process and is time sortable in integer and base32hex representation. The 128-bit version is a K-sortable, globally unique identifier for use in distributed systems and is compatible with UUID.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core as ic\n&gt;&gt;&gt; ic.gen_flake_code(bits=64)\n{'iscc': 'ISCC:OAAQC7YN7PG2XOR4'}\n&gt;&gt;&gt; ic.gen_flake_code(bits=128)\n{'iscc': 'ISCC:OABQC7YN7RJGUUTLKDSKBXO25MA5E'}\n# Or use the convenience Flake class for easy access to different representations\n&gt;&gt;&gt; flake = ic.Flake(bits=64)\n&gt;&gt;&gt; flake.iscc\n'ISCC:OAAQC7YOADBZYNF7'\n&gt;&gt;&gt; flake.time\n'2022-02-18T18:03:25.468'\n&gt;&gt;&gt; flake.int\n107820312524764351\n&gt;&gt;&gt; flake.string\n'05VGS063JGQBU'\n</code></pre>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code","title":"<code>gen_flake_code(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest standard algorithm</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code_v0","title":"<code>gen_flake_code_v0(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest algorithm v0</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.uid_flake_v0","title":"<code>uid_flake_v0(ts = None, bits = ic.core_opts.flake_bits)</code>","text":"<p>Generate time and randomness based Flake-Hash</p> <p>Parameters:</p> Name Type Description Default <code>ts</code> <code>Optional[float]</code> <p>Unix timestamp (defaults to current time)</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length resulting Flake-Code (multiple of 32)</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Flake-Hash digest</p>"},{"location":"units/code_instance/","title":"ISCC - Instance-Code","text":"<p>A data checksum.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code","title":"<code>gen_instance_code(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with properties: iscc, datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code_v0","title":"<code>gen_instance_code_v0(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.hash_instance_v0","title":"<code>hash_instance_v0(stream)</code>","text":"<p>Create 256-bit hash digest for the Instance-Code body</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for hash generation.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Instance-Hash digest used as body of Instance-Code</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0","title":"<code>InstanceHasherV0</code>","text":"<p>Incremental Instance-Hash generator.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Instance-Hash generator.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Data</code> <p>Data to be hashed</p> required"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.digest","title":"<code>digest()</code>","text":"<p>Return Instance-Hash</p> <p>Returns:</p> Type Description <code>bytes</code> <p>Instance-Hash digest</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.multihash","title":"<code>multihash()</code>","text":"<p>Return blake3 multihash</p> <p>Returns:</p> Type Description <code>str</code> <p>Blake3 hash as 256-bit multihash</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.code","title":"<code>code(bits = ic.core_opts.instance_bits)</code>","text":"<p>Encode digest as an ISCC Instance-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Instance-Code</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Instance-Code</p>"},{"location":"units/code_meta/","title":"ISCC - Meta-Code","text":"<p>A similarity preserving hash for digital asset metadata.</p>"},{"location":"units/code_meta/#purpose","title":"Purpose","text":"<p>The Meta-Code is the first possible (optional) unit of an ISCC-CODE. It is calculated from the metadata of a digital asset. The primary purpose of the Meta-Code is to aid the discovery of digital assets with similar metadata and the detection of metadata anomalies. As a secondary function, Meta-Code processing also creates a secure Meta-Hash for cryptogrpahic binding purposes.</p>"},{"location":"units/code_meta/#inputs","title":"Inputs","text":"<p>The metadata supplied for Meta-Code calculation is called Seed-Metadata. Seed-Metadata has 3 possible elements:</p> <ul> <li>name (required): The name or title of the work manifested by the digital asset.</li> <li>description (optional): A disambiguating textual description of the digital asset.</li> <li>meta (optional): Industry-sector or use-case specific metadata, encoded as     Data-URL (RFC-2397).</li> </ul> <p>Note</p> <p>Due to the broad applicability of the ISCC we do not prescribe a particular schema for the <code>meta</code>-element. Instead we use the Data-URL format because it can encode and self-describe any conceivable metadata in a sufficently machine-interpretable form at any desired specificity.</p> <p>Data-URL Examples:</p> <ul> <li>Metadata is \"some\" JSON: <code>data:application/json;base64,&lt;data&gt;</code></li> <li>Metadata is JSON-LD: <code>data:application/ld+json;base64,&lt;data&gt;</code></li> <li>Metadata is \"some\" XML: <code>data:application/xml;base64,&lt;data&gt;</code></li> <li>Metadata is MARC21 XML: <code>data:application/marcxml+xml;base64,&lt;data&gt;</code></li> <li>Metadata is IPTC NewsML: <code>data:application/vnd.iptc.g2.newsitem+xml;base64,&lt;data&gt;</code></li> </ul> <p>Data-URLs are also supported by all major internet browsers.</p>"},{"location":"units/code_meta/#processing","title":"Processing","text":""},{"location":"units/code_meta/#meta-code","title":"Meta-Code","text":"<p>The first 32-bits of a Meta-Code are calculated as a simliarity hash from the <code>name</code> field. The second 32-bits are also calculated from the <code>name</code> field if no other input was supplied. If <code>description</code> is suplied but no <code>meta</code>, the <code>description</code> will be used for the second 32-bits. If <code>meta</code> is supplied it will be used in favour of <code>description</code> for the second 32-bits.</p> <pre><code>flowchart LR\n    B{name?}\n    B --&gt;|Yes| J[\"P1 = SH(name)\"]\n    J --&gt; D{meta?}\n    D --&gt;|Yes| F[\"Meta-Code = P1 + SH(meta)\"]\n    D --&gt;|No| G{description?}\n    G --&gt;|Yes| H[\"Meta-Code = P1 + SH(description)\"]\n    G --&gt;|No| I[\"Meta-Code = P1\"]\n    B --&gt;|No| E[Skip Meta-Code]</code></pre> <p>Note</p> <p>To support automation and reproducibility, applications that generate ISCCs, should prioritize metadata that is automatically extracted from the digital asset.</p> <p>If embedded metadata is not available or known to be unreliable an application should rely on external metadata or explicitly ask users to supply at least the <code>name</code>-field. Applications should then first embed metadata into the asset before calculating the ISCC-CODE. This ensures that the embedded metadata is bound to the asset by the Instance-Code.</p> <p>If neither embedded nor external metadata is available, the application may resort to use the filename of the digital asset as value for the <code>name</code>-field. If no value can be determined for the <code>name</code>-field, an application shall skip generation of a Meta-Code and create an ISCC-CODE without a Meta-Code.</p>"},{"location":"units/code_meta/#meta-hash","title":"Meta-Hash","text":"<p>In addition to the Meta-Code we also create a cryptographic hash (the Meta-Hash) of the supplied Seed-Metadata. It is used to securely bind metadata to the digital asset.</p> <pre><code>flowchart LR\n    N{name?}\n    N --&gt;|Yes| A{meta?}\n    N --&gt;|No| E[Skip Meta-Hash]\n    A --&gt;|Yes| H[\"Meta-Hash = H(meta)\"]\n    A --&gt;|No| B{description?}\n    B --&gt;|Yes| HND[\"Meta-Hash = H(name + description)\"]\n    B --&gt;|No| HN[\"Meta-Hash = H(name)\"]</code></pre>"},{"location":"units/code_meta/#functions","title":"Functions","text":""},{"location":"units/code_meta/#iscc_core.code_meta.gen_meta_code_v0","title":"<code>gen_meta_code_v0(name, description = None, meta = None, bits = ic.core_opts.meta_bits)</code>","text":"<p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.soft_hash_meta_v0","title":"<code>soft_hash_meta_v0(name, extra = None)</code>","text":"<p>Calculate simmilarity preserving 256-bit hash digest from asset metadata.</p> <p>Textual input should be stripped of markup, normalized and trimmed before hashing. Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be serialized in a canonical form (for example JCS for JSON)</p> <p>Note</p> <p>The processing algorithm depends on the type of the <code>extra</code> input. If the <code>extra</code> field is supplied and non-empty, we create separate hashes for <code>name</code> and <code>extra</code> and interleave them in 32-bit chunks:</p> <ul> <li> <p>If the <code>extra</code> input is <code>None</code> or an empty <code>str</code>/<code>bytes</code> object the Meta-Hash will be generated from the <code>name</code>-field only.</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty text string (str) the string is lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty bytes object the processing is done bytewise.</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Title of the work manifested in the digital asset</p> required <code>extra</code> <code>Union[str,bytes,None]</code> <p>Additional metadata for disambiguation</p> <code>None</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit simhash digest for Meta-Code</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def soft_hash_meta_v0(name, extra=None):\n# type: (str, Union[str,bytes,None]) -&gt; bytes\n\"\"\"\n    Calculate simmilarity preserving 256-bit hash digest from asset metadata.\n    Textual input should be stripped of markup, normalized and trimmed before hashing.\n    Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be\n    serialized in a canonical form (for example\n    [JCS](https://tools.ietf.org/id/draft-rundgren-json-canonicalization-scheme-00.html) for JSON)\n    !!! note\n        The processing algorithm depends on the type of the `extra` input.\n        If the `extra` field is supplied and non-empty, we create separate hashes for\n        `name` and `extra` and interleave them in 32-bit chunks:\n        - If the `extra` input is `None` or an empty `str`/`bytes` object the Meta-Hash will\n        be generated from the `name`-field only.\n        - If the `extra`-input is a non-empty **text** string (str) the string is\n        lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).\n        - If the `extra`-input is a non-empty **bytes** object the processing is done bytewise.\n    :param str name: Title of the work manifested in the digital asset\n    :param Union[str,bytes,None] extra: Additional metadata for disambiguation\n    :return: 256-bit simhash digest for Meta-Code\n    :rtype: bytes\n    \"\"\"\nname = ic.text_collapse(name)\nname_n_grams = ic.sliding_window(name, width=ic.core_opts.meta_ngram_size_text)\nname_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in name_n_grams]\nsimhash_digest = ic.alg_simhash(name_hash_digests)\nif extra in {None, \"\", b\"\"}:\nreturn simhash_digest\nelse:\n# Augment with interleaved hash for extra metadata\nif isinstance(extra, bytes):\n# Raw bytes are handled per byte\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_bytes)\nextra_hash_digests = [blake3(ngram).digest() for ngram in extra_n_grams]\nelif isinstance(extra, str):\n# Text is collapsed and handled per character (multibyte)\nextra = ic.text_collapse(extra)\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_text)\nextra_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in extra_n_grams]\nelse:\nraise ValueError(\"parameter `extra` must be of type str or bytes!\")\nextra_simhash_digest = ic.alg_simhash(extra_hash_digests)\n# Interleave first half of name and extra simhashes in 32-bit chunks\nchunks_simhash_digest = sliced(simhash_digest[:16], 4)\nchunks_extra_simhash_digest = sliced(extra_simhash_digest[:16], 4)\ninterleaved = interleave(chunks_simhash_digest, chunks_extra_simhash_digest)\nsimhash_digest = bytearray()\nfor chunk in interleaved:\nsimhash_digest += chunk\nsimhash_digest = bytes(simhash_digest)\nreturn simhash_digest\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_clean","title":"<code>text_clean(text)</code>","text":"<p>Clean text for display.</p> <ul> <li>Normalize with NFKC normalization.</li> <li>Remove Control Characters (except newlines)</li> <li>Reduce multiple consecutive newlines to a maximum of two newlines</li> <li>Strip leading and trailing whitespace</li> </ul> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_clean(text):\n# type: (str) -&gt; str\n\"\"\"\n    Clean text for display.\n    - Normalize with NFKC normalization.\n    - Remove Control Characters (except newlines)\n    - Reduce multiple consecutive newlines to a maximum of two newlines\n    - Strip leading and trailing whitespace\n    \"\"\"\n# Unicode normalize\ntext = unicodedata.normalize(\"NFKC\", text)\n# Remove control characters\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] != \"C\" or ch in ic.core_opts.text_newlines\n)\n# Collapse more than two consecutive newlines\nchars = []\nnewline_count = 0\nfor c in text:\nif c in ic.core_opts.text_newlines:\nif newline_count &lt; 2:\nchars.append(\"\\n\")\nnewline_count += 1\ncontinue\nelse:\nnewline_count = 0\nchars.append(c)\ntext = \"\".join(chars)\nreturn text.strip()\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_remove_newlines","title":"<code>text_remove_newlines(text)</code>","text":"<p>Remove newlines.</p> <p>The <code>name</code> field serves as a displayable title. We remove newlines and leading and trailing whitespace. We also collapse consecutive spaces to single spaces.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <p>Text for newline removal</p> required <p>Returns:</p> Type Description <code>str</code> <p>Single line of text</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_remove_newlines(text):\n# type: (str) -&gt; str\n\"\"\"\n    Remove newlines.\n    The `name` field serves as a displayable title. We remove newlines and leading and trailing\n    whitespace. We also collapse consecutive spaces to single spaces.\n    :param text: Text for newline removal\n    :return: Single line of text\n    :rtype: str\n    \"\"\"\nreturn \" \".join(text.split())\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_trim","title":"<code>text_trim(text, nbytes)</code>","text":"<p>Trim text such that its utf-8 encoded size does not exceed <code>nbytes</code>.</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_trim(text, nbytes):\n# type: (str, int) -&gt; str\n\"\"\"Trim text such that its utf-8 encoded size does not exceed `nbytes`.\"\"\"\nreturn text.encode(\"utf-8\")[:nbytes].decode(\"utf-8\", \"ignore\").strip()\n</code></pre>"},{"location":"units/content/","title":"ISCC - Content-Codes","text":""},{"location":"units/content/code_content_audio/","title":"ISCC - Audio-Code","text":"<p>A similarity preserving hash for audio content (soft hash).</p> <p>Creates an ISCC object that provides an <code>iscc</code>-field with an Audio-Code and a <code>duration</code>-field.</p> <p>The Content-Code Audio is generated from a Chromaprint fingerprint provided as a vector of 32-bit signed integers. The iscc-sdk uses fpcalc to extract Chromaprint vectors with the following command line parameters:</p> <p><code>$ fpcalc -raw -json -signed -length 0 myaudiofile.mp3</code></p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code","title":"<code>gen_audio_code(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code_v0","title":"<code>gen_audio_code_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.soft_hash_audio_v0","title":"<code>soft_hash_audio_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create audio similarity hash from a chromaprint vector.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting similarity hash (multiple of 32)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Audio-Hash digest</p>"},{"location":"units/content/code_content_image/","title":"ISCC - Image-Code","text":"<p>A similarity preserving perceptual hash for images.</p> <p>The ISCC Content-Code Image is created by calculating a discrete cosine transform on normalized image-pixels and comparing the values from the upper left area of the dct-matrix against their median values to set the hash-bits.</p> <p>Images must be normalized before using gen_image_code. Prepare images as follows:</p> <ul> <li>Transpose image according to EXIF Orientation</li> <li>Add white background to image if it has alpha transparency</li> <li>Crop empty borders of image</li> <li>Convert image to grayscale</li> <li>Resize image to 32x32</li> <li>Flatten 32x32 matrix to an array of 1024 grayscale (uint8) pixel values</li> </ul>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code","title":"<code>gen_image_code(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values).</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code_v0","title":"<code>gen_image_code_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.soft_hash_image_v0","title":"<code>soft_hash_image_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Calculate image hash from normalized grayscale pixel sequence of length 1024.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> required <code>bits</code> <code>int</code> <p>Bit-length of image hash (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving Image-Hash digest.</p>"},{"location":"units/content/code_content_mixed/","title":"ISCC - Mixed Code","text":"<p>A similarity hash for mixed media content.</p> <p>Creates an ISCC object that provides a <code>iscc</code>-field a Mixed-Code and a <code>parts</code>-field that lists the input codes.</p> <p>Many digital assets embed multiple assets of different mediatypes in a single file. Text documents may include images, video includes audio in most cases. The ISCC Content-Code-Mixed encodes the similarity of a collection of assets of the same or different mediatypes that may occur in a multimedia asset.</p> <p>Applications that create mixed Content-Codes must be capable to extract embedded assets and create individual Content-Codes per asset.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code","title":"<code>gen_mixed_code(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code Mixed with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code_v0","title":"<code>gen_mixed_code_v0(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.soft_hash_codes_v0","title":"<code>soft_hash_codes_v0(cc_digests, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create a similarity hash from multiple Content-Code digests.</p> <p>The similarity hash is created from the bodies of the input codes with the first byte of the code-header prepended.</p> <p>All codes must be of main-type CONTENT and have a minimum length of <code>bits</code>.</p> <p>Parameters:</p> Name Type Description Default <code>cc_digests</code> <code>Sequence[bytes]</code> <p>a list of Content-Code digests.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving byte hash.</p>"},{"location":"units/content/code_content_text/","title":"ISCC - Text Code","text":"<p>A similarity preserving hash for plain-text content (soft hash).</p> <p>The ISCC Text-Code is generated from plain-text that has been extracted from a media assets.</p> <p>Warning</p> <p>Plain-text extraction from documents in various formats (especially PDF) may yield very diffent results depending on the extraction tools being used. The iscc-sdk uses Apache Tika to extract text from documents for Text-Code generation.</p> <p>Algorithm overview</p> <ul> <li>Apply <code>text_collapse</code> function to text input</li> <li>Count characters of collapsed text</li> <li>Apply <code>soft_hash_text_v0</code> to collapsed text</li> </ul>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.gen_text_code_v0","title":"<code>gen_text_code_v0(text, bits = ic.core_opts.text_bits)</code>","text":"<p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.text_collapse","title":"<code>text_collapse(text)</code>","text":"<p>Normalize and simplify text for similarity hashing.</p> <ul> <li>Decompose with NFD normalization.</li> <li>Remove all whitespace characters and convert text to lower case</li> <li>Filter control characters, marks (diacritics), and punctuation</li> <li>Recombine with NFKC normalization.</li> </ul> <p>Note</p> <p>See: Unicode normalization.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be collapsed.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Collapsed plain text.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def text_collapse(text):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize and simplify text for similarity hashing.\n    - Decompose with NFD normalization.\n    - Remove all whitespace characters and convert text to lower case\n    - Filter control characters, marks (diacritics), and punctuation\n    - Recombine with NFKC normalization.\n    !!! note\n        See: [Unicode normalization](https://unicode.org/reports/tr15/).\n    :param str text: Plain text to be collapsed.\n    :return: Collapsed plain text.\n    :rtype: str\n    \"\"\"\n# Decompose with NFD\ntext = unicodedata.normalize(\"NFD\", text)\n# Remove all whitespace and convert text to lower case\ntext = \"\".join(text.split()).lower()\n# Filter control characters, marks (diacritics), and punctuation\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] not in ic.core_opts.text_unicode_filter\n)\n# Recombine\ntext = unicodedata.normalize(\"NFKC\", text)\nreturn text\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.soft_hash_text_v0","title":"<code>soft_hash_text_v0(text)</code>","text":"<p>Creates a 256-bit similarity preserving hash for text input with algorithm v0.</p> <ul> <li>Slide over text with a   <code>text_ngram_size</code> wide window   and create <code>xxh32</code> hashes</li> <li>Create a <code>minhash_256</code> from the hashes generated   in the previous step.</li> </ul> <p>Note</p> <p>Before passing text to this function it must be:</p> <ul> <li>stripped of markup</li> <li>normalized</li> <li>stripped of whitespace</li> <li>lowercased</li> </ul> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be hashed.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit similarity preserving byte hash.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def soft_hash_text_v0(text):\n# type: (str) -&gt; bytes\n\"\"\"\n    Creates a 256-bit similarity preserving hash for text input with algorithm v0.\n    - Slide over text with a\n      [`text_ngram_size`][iscc_core.options.CoreOptions.text_ngram_size] wide window\n      and create [`xxh32`](https://cyan4973.github.io/xxHash/) hashes\n    - Create a [`minhash_256`][iscc_core.minhash.alg_minhash_256] from the hashes generated\n      in the previous step.\n    !!! note\n        Before passing text to this function it must be:\n        - stripped of markup\n        - normalized\n        - stripped of whitespace\n        - lowercased\n    :param str text: Plain text to be hashed.\n    :return: 256-bit similarity preserving byte hash.\n    :rtype: bytes\n    \"\"\"\nngrams = ic.sliding_window(text, ic.core_opts.text_ngram_size)\nfeatures = [xxhash.xxh32_intdigest(s.encode(\"utf-8\")) for s in ngrams]\nhash_digest = ic.alg_minhash_256(features)\nreturn hash_digest\n</code></pre>"},{"location":"units/content/code_content_video/","title":"ISCC - Video-Code","text":"<p>A similarity preserving hash for video content</p> <p>The Content-Code Video is generated from MPEG-7 video frame signatures. The iscc-sdk uses ffmpeg to extract frame signatures with the following command line parameters:</p> <p><code>$ ffmpeg -i video.mpg -vf fps=fps=5,signature=format=xml:filename=sig.xml -f null -</code></p> <p>The relevant frame signatures can be parsed from the following elements in sig.xml:</p> <p><code>&lt;FrameSignature&gt;0  0  0  1  0  0  1  0  1  1  0  0  1  1 ...&lt;/FrameSignature&gt;</code></p> <p>Tip</p> <p>It is also possible to extract the signatures in a more compact binary format. But the format requires a custom binary parser to decode the frame signaturs.</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code","title":"<code>gen_video_code(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code_v0","title":"<code>gen_video_code_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.soft_hash_video_v0","title":"<code>soft_hash_video_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Compute video hash v0 from MP7 frame signatures.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>2D matrix of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code>"},{"location":"utilities/utils/","title":"ISCC - Utilities","text":""},{"location":"utilities/utils/#iscc_core.utils.json_canonical","title":"<code>json_canonical(obj)</code>","text":"<p>Canonical, deterministic serialization of ISCC metadata.</p> <p>We serialize ISCC metadata in a deterministic/reproducible manner by using JCS (RFC 8785) canonicalization.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def json_canonical(obj):\n# type: (Any) -&gt; bytes\n\"\"\"\n    Canonical, deterministic serialization of ISCC metadata.\n    We serialize ISCC metadata in a deterministic/reproducible manner by using\n    [JCS (RFC 8785)](https://datatracker.ietf.org/doc/html/rfc8785) canonicalization.\n    \"\"\"\nser = jcs.canonicalize(obj)\ndes = json.loads(ser)\nif des != obj:\nraise ValueError(f\"Not canonicalizable {obj} round-trips to {des}\")\nreturn ser\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.sliding_window","title":"<code>sliding_window(seq, width)</code>","text":"<p>Generate a sequence of equal \"width\" slices each advancing by one elemnt.</p> <p>All types that have a length and can be sliced are supported (list, tuple, str ...). The result type matches the type of the input sequence. Fragment slices smaller than the width at the end of the sequence are not produced. If \"witdh\" is smaller than the input sequence than one element will be returned that is shorter than the requested width.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>Sequence</code> <p>Sequence of values to slide over</p> required <code>width</code> <code>int</code> <p>Width of sliding window in number of items</p> required <p>Returns:</p> Type Description <code>Generator</code> <p>A generator of window sized items</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def sliding_window(seq, width):\n# type: (Sequence, int) -&gt; Generator\n\"\"\"\n    Generate a sequence of equal \"width\" slices each advancing by one elemnt.\n    All types that have a length and can be sliced are supported (list, tuple, str ...).\n    The result type matches the type of the input sequence.\n    Fragment slices smaller than the width at the end of the sequence are not produced.\n    If \"witdh\" is smaller than the input sequence than one element will be returned that\n    is shorter than the requested width.\n    :param Sequence seq: Sequence of values to slide over\n    :param int width: Width of sliding window in number of items\n    :returns: A generator of window sized items\n    :rtype: Generator\n    \"\"\"\nif width &lt; 2:\nraise AssertionError(\"Sliding window width must be 2 or bigger.\")\nidx = range(max(len(seq) - width + 1, 1))\nreturn (seq[i : i + width] for i in idx)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_compare","title":"<code>iscc_compare(a, b)</code>","text":"<p>Calculate separate hamming distances of compatible components of two ISCCs</p> <p>Returns:</p> Type Description <code>dict</code> <p>A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_compare(a, b):\n# type: (str, str) -&gt; dict\n\"\"\"\n    Calculate separate hamming distances of compatible components of two ISCCs\n    :return: A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match\n    :rtype: dict\n    \"\"\"\nac = [ic.Code(unit) for unit in ic.iscc_decompose(a)]\nbc = [ic.Code(unit) for unit in ic.iscc_decompose(b)]\nresult = {}\nfor ca in ac:\nfor cb in bc:\ncat = (ca.maintype, ca.subtype, ca.version)\ncbt = (cb.maintype, cb.subtype, ca.version)\nif cat == cbt:\nif ca.maintype != ic.MT.INSTANCE:\nresult[ca.maintype.name.lower() + \"_dist\"] = iscc_distance_bytes(\nca.hash_bytes, cb.hash_bytes\n)\nelse:\nresult[\"instance_match\"] = ca.hash_bytes == cb.hash_bytes\nreturn result\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_similarity","title":"<code>iscc_similarity(a, b)</code>","text":"<p>Calculate similarity of ISCC codes as a percentage value (0-100).</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Similarity of ISCC a and b in percent (based on hamming distance)</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_similarity(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate similarity of ISCC codes as a percentage value (0-100).\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Similarity of ISCC a and b in percent (based on hamming distance)\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nhdist = iscc_distance_bytes(a, b)\nnbits = len(a) * 8\nsim = int(((nbits - hdist) / nbits) * 100)\nreturn sim\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance","title":"<code>iscc_distance(a, b)</code>","text":"<p>Calculate hamming distance of ISCC codes.</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distanced in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate hamming distance of ISCC codes.\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Hamming distanced in number of bits.\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nreturn iscc_distance_bytes(a, b)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance_bytes","title":"<code>iscc_distance_bytes(a, b)</code>","text":"<p>Calculate hamming distance for binary hash digests of equal length.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <code>bytes</code> <p>binary hash digest</p> required <code>b</code> <code>bytes</code> <p>binary hash digest</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distance in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance_bytes(a, b):\n# type: (bytes, bytes) -&gt; int\n\"\"\"\n    Calculate hamming distance for binary hash digests of equal length.\n    :param bytes a: binary hash digest\n    :param bytes b: binary hash digest\n    :return: Hamming distance in number of bits.\n    :rtype: int\n    \"\"\"\nif len(a) != len(b):\nraise AssertionError(f\"Hash diggest of unequal length: {len(a)} vs {len(b)}\")\nba, bb = bitarray(), bitarray()\nba.frombytes(a)\nbb.frombytes(b)\nreturn count_xor(ba, bb)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_pair_unpack","title":"<code>iscc_pair_unpack(a, b)</code>","text":"<p>Unpack two ISCC codes and return their body hash digests if their headers match.</p> <p>Headers match if their MainType, SubType, and Version are identical.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>Tuple[bytes, bytes]</code> <p>Tuple with hash digests of a and b</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If ISCC headers don\u00b4t match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_pair_unpack(a, b):\n# type: (str, str) -&gt; Tuple[bytes, bytes]\n\"\"\"\n    Unpack two ISCC codes and return their body hash digests if their headers match.\n    Headers match if their MainType, SubType, and Version are identical.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Tuple with hash digests of a and b\n    :rtype: Tuple[bytes, bytes]\n    :raise ValueError: If ISCC headers don\u00b4t match\n    \"\"\"\na, b = ic.iscc_clean(ic.iscc_normalize(a)), ic.iscc_clean(ic.iscc_normalize(b))\na, b = ic.decode_base32(a), ic.decode_base32(b)\na, b = ic.decode_header(a), ic.decode_header(b)\nif not a[:-1] == b[:-1]:\nraise ValueError(f\"ISCC headers don\u00b4t match: {a}, {b}\")\nreturn a[-1], b[-1]\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"ISCC - Codec &amp; Algorithms","text":"<p><code>iscc-core</code> is the reference implementation of the core algorithms of the ISCC (International Standard Content Code)</p>"},{"location":"#what-is-the-iscc","title":"What is the ISCC","text":"<p>The ISCC is a similarity preserving fingerprint and identifier for digital media assets.</p> <p>ISCCs are generated algorithmically from digital content, just like cryptographic hashes. However, instead of using a single cryptographic hash function to identify data only, the ISCC uses various algorithms to create a composite identifier that exhibits similarity-preserving properties (soft hash).</p> <p>The component-based structure of the ISCC identifies content at multiple levels of abstraction. Each component is self-describing, modular, and can be used separately or with others to aid in various content identification tasks. The algorithmic design supports content deduplication, database synchronization, indexing, integrity verification, timestamping, versioning, data provenance, similarity clustering, anomaly detection, usage tracking, allocation of royalties, fact-checking and general digital asset management use-cases.</p>"},{"location":"#what-is-iscc-core","title":"What is <code>iscc-core</code>","text":"<p><code>iscc-core</code> is the python based reference implementation of the ISCC core algorithms as defined by ISO 24138. It also a good reference for porting ISCC to other programming languages.</p> <p>Tip</p> <p>This is a low level reference implementation that does not inlcude features like mediatype detection, metadata extraction or file format specific content extraction. Please have a look at iscc-sdk which adds those higher level features on top of the <code>iscc-core</code> library.</p>"},{"location":"#implementors-guide","title":"Implementors Guide","text":""},{"location":"#reproducible-environment","title":"Reproducible Environment","text":"<p>For reproducible installation of the reference implementation we included a <code>poetry.lock</code> file with pinned dependencies. Install them using Python Poetry with the command <code>poetry install</code> in the root folder.</p>"},{"location":"#repository-structure","title":"Repository structure","text":"<pre><code>iscc-core\n\u251c\u2500\u2500 docs       # Markdown and other assets for mkdocs documentation\n\u251c\u2500\u2500 examples   # Example scripts using the reference code\n\u251c\u2500\u2500 iscc_core  # Actual source code of the reference implementation\n\u251c\u2500\u2500 tests      # Tests for the reference implementation\n\u2514\u2500\u2500 tools      # Development tools\n</code></pre>"},{"location":"#testing-conformance","title":"Testing &amp; Conformance","text":"<p>The reference implementation comes with 100% test coverage. To run the conformance selftest from the repository root use <code>poetry run python -m iscc_core</code>. To run the complete test suite use <code>poetry run pytest</code>.</p> <p>To build a conformant implementation work through the follwing top level entrypoint functions:</p> <pre><code>gen_meta_code_v0\ngen_text_code_v0\ngen_image_code_v0\ngen_audio_code_v0\ngen_video_code_v0\ngen_mixed_code_v0\ngen_data_code_v0\ngen_instance_code_v0\ngen_iscc_code_v0\n</code></pre> <p>The corresponding test vectors can be found in <code>iscc_core/data.json</code>.</p>"},{"location":"#iscc-architecture","title":"ISCC Architecture","text":""},{"location":"#iscc-maintypes","title":"ISCC MainTypes","text":"Idx Slug Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more components with common header"},{"location":"#installation","title":"Installation","text":"<p>Use the package manager pip to install <code>iscc-core</code> as a library.</p> <pre><code>pip install iscc-core\n</code></pre>"},{"location":"#quick-start","title":"Quick Start","text":"<pre><code>import json\nimport iscc_core as ic\nmeta_code = ic.gen_meta_code(name=\"ISCC Test Document!\")\nprint(f\"Meta-Code:     {meta_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(meta_code['iscc'])}\\n\")\n# Extract text from file\nwith open(\"demo.txt\", \"rt\", encoding=\"utf-8\") as stream:\ntext = stream.read()\ntext_code = ic.gen_text_code_v0(text)\nprint(f\"Text-Code:     {text_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(text_code['iscc'])}\\n\")\n# Process raw bytes of textfile\nwith open(\"demo.txt\", \"rb\") as stream:\ndata_code = ic.gen_data_code(stream)\nprint(f\"Data-Code:     {data_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(data_code['iscc'])}\\n\")\nstream.seek(0)\ninstance_code = ic.gen_instance_code(stream)\nprint(f\"Instance-Code: {instance_code['iscc']}\")\nprint(f\"Structure:     {ic.iscc_explain(instance_code['iscc'])}\\n\")\n# Combine ISCC-UNITs into ISCC-CODE\niscc_code = ic.gen_iscc_code(\n(meta_code[\"iscc\"], text_code[\"iscc\"], data_code[\"iscc\"], instance_code[\"iscc\"])\n)\n# Create convenience `Code` object from ISCC string\niscc_obj = ic.Code(iscc_code[\"iscc\"])\nprint(f\"ISCC-CODE:     {ic.iscc_normalize(iscc_obj.code)}\")\nprint(f\"Structure:     {iscc_obj.explain}\")\nprint(f\"Multiformat:   {iscc_obj.mf_base32}\\n\")\n# Compare with changed ISCC-CODE:\nnew_dc, new_ic = ic.Code.rnd(mt=ic.MT.DATA), ic.Code.rnd(mt=ic.MT.INSTANCE)\nnew_iscc = ic.gen_iscc_code((meta_code[\"iscc\"], text_code[\"iscc\"], new_dc.uri, new_ic.uri))\nprint(f\"Compare ISCC-CODES:\\n{iscc_obj.uri}\\n{new_iscc['iscc']}\")\nprint(json.dumps(ic.iscc_compare(iscc_obj.code, new_iscc[\"iscc\"]), indent=2))\n</code></pre> <p>The output of this example is as follows:</p> <pre><code>Meta-Code:     ISCC:AAAT4EBWK27737D2\nStructure:     META-NONE-V0-64-3e103656bffdfc7a\n\nText-Code:     ISCC:EAAQMBEYQF6457DP\nStructure:     CONTENT-TEXT-V0-64-060498817dcefc6f\n\nData-Code:     ISCC:GAA7UJMLDXHPPENG\nStructure:     DATA-NONE-V0-64-fa258b1dcef791a6\n\nInstance-Code: ISCC:IAA3Y7HR2FEZCU4N\nStructure:     INSTANCE-NONE-V0-64-bc7cf1d14991538d\n\nISCC-CODE:     ISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nStructure:     ISCC-TEXT-V0-MCDI-3e103656bffdfc7a060498817dcefc6ffa258b1dcef791a6bc7cf1d14991538d\nMultiformat:   bzqavabj6ca3fnp757r5ambeyqf6457dp7isywhoo66i2npd46hiutektru\n\nCompare ISCC-CODES:\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G76RFRMO4554RU26HZ4ORJGIVHDI\nISCC:KACT4EBWK27737D2AYCJRAL5Z36G7Y7HA2BMECKMVRBEQXR2BJOS6NA\n{\n  \"meta_dist\": 0,\n  \"content_dist\": 0,\n  \"data_dist\": 33,\n  \"instance_match\": false\n}\n</code></pre>"},{"location":"#documentation","title":"Documentation","text":"<p>Documentation is published at https://core.iscc.codes</p>"},{"location":"#development","title":"Development","text":"<p>Requirements</p> <ul> <li>Python 3.7.2 or higher for code generation and static site building.</li> <li>Poetry for installation and dependency management.</li> </ul> <p>Development Setup</p> <pre><code>git clone https://github.com/iscc/iscc-core.git\ncd iscc-core\npoetry install\n</code></pre> <p>Development Tasks</p> <p>Tests, coverage, code formatting and other tasks can be run with the <code>poe</code> command:</p> <pre><code>poe\n\nPoe the Poet - A task runner that works well with poetry.\nversion 0.18.1\n\nResult: No task specified.\n\nUSAGE\n  poe [-h] [-v | -q] [--root PATH] [--ansi | --no-ansi] task [task arguments]\nGLOBAL OPTIONS\n  -h, --help     Show this help page and exit\n--version      Print the version and exit\n-v, --verbose  Increase command output (repeatable)\n-q, --quiet    Decrease command output (repeatable)\n-d, --dry-run  Print the task contents but don't actually run it\n  --root PATH    Specify where to find the pyproject.toml\n  --ansi         Force enable ANSI output\n  --no-ansi      Force disable ANSI output\nCONFIGURED TASKS\n  gentests       Generate conformance test data\n  format         Code style formating with black\n  docs           Copy README.md to /docs\n  format-md      Markdown formating with mdformat\n  lf             Convert line endings to lf\n  test           Run tests with coverage\n  sec            Security check with bandit\n  all\n</code></pre> <p>Use <code>poe all</code> to run all tasks before committing any changes.</p>"},{"location":"#maintainers","title":"Maintainers","text":"<p>@titusz</p>"},{"location":"#contributing","title":"Contributing","text":"<p>Pull requests are welcome. For significant changes, please open an issue first to discuss your plans. Please make sure to update tests as appropriate.</p> <p>You may also want join our developer chat on Telegram at https://t.me/iscc_dev.</p>"},{"location":"changelog/","title":"Changelog","text":""},{"location":"changelog/#109-2024-03-17","title":"[1.0.9] - 2024-03-17","text":"<ul> <li>Updated dependencies</li> <li>Fixed issue with decoding lower case base32hex strings</li> <li>Added support for MSCDI validation</li> </ul>"},{"location":"changelog/#108-2024-01-30","title":"[1.0.8] - 2024-01-30","text":"<ul> <li>Added implementors guide to README.md</li> <li>Improved pydantic v2 import logic</li> <li>Improved prefix extraction during normalization</li> <li>Improved canonical ISCC string validation</li> </ul>"},{"location":"changelog/#107-2024-01-07","title":"[1.0.7] - 2024-01-07","text":"<ul> <li>Support pydantic v1 &amp; v2</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#106-2023-12-15","title":"[1.0.6] - 2023-12-15","text":"<ul> <li>Added wheel package testing to CI</li> <li>Fixed incompatible dependencies</li> </ul>"},{"location":"changelog/#105-2023-12-07","title":"[1.0.5] - 2023-12-07","text":"<ul> <li>Improved simhash performance</li> <li>Added native dct and wtahash support</li> <li>Added Python 3.12 support</li> <li>Updated and relax dependencies</li> </ul>"},{"location":"changelog/#104-2023-06-05","title":"[1.0.4] - 2023-06-05","text":"<ul> <li>Removed bases dependency</li> <li>Fixed mkdocstrings</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#103-2023-03-12","title":"[1.0.3] - 2023-03-12","text":"<ul> <li>Fix binary wheels</li> </ul>"},{"location":"changelog/#102-2023-03-12","title":"[1.0.2] - 2023-03-12","text":"<ul> <li>Publish binary wheels</li> </ul>"},{"location":"changelog/#101-2023-03-11","title":"[1.0.1] - 2023-03-11","text":"<ul> <li>Switch to standard bitarray module</li> <li>Switch tests to latest environments</li> <li>Add Python 3.11 support to TROVE classifiers</li> </ul>"},{"location":"changelog/#100-2023-01-24","title":"[1.0.0] - 2023-01-24","text":"<ul> <li>ISO/CD 24138 v1 Release</li> <li>Updateted dependencies</li> </ul>"},{"location":"changelog/#0214-2023-01-17","title":"[0.2.14] - 2023-01-17","text":"<ul> <li>Added ISO Reference documentation</li> <li>Removed non-standard conformance tests</li> <li>Fixed Meta-Code documentation</li> <li>Improved documentation CSS</li> </ul>"},{"location":"changelog/#0213-2023-01-16","title":"[0.2.13] - 2023-01-16","text":"<ul> <li>Added documentation for options</li> <li>Added python 3.11 support</li> <li>Added Markdown formating</li> <li>Added developmnet documentation</li> <li>Updated architecture figures</li> <li>Documentation cleanup</li> <li>Improved example code</li> <li>Optimized LF conversion</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0212-2022-11-24","title":"[0.2.12] - 2022-11-24","text":"<ul> <li>Fixed issue with data url compound media types</li> <li>Added ISCC version validation to <code>iscc_validate</code></li> <li>Added prefix check to <code>iscc_normalize</code></li> <li>Bundled fonts with documentation</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#0211-2022-07-03","title":"[0.2.11] - 2022-07-03","text":"<ul> <li>Add support for gracefull build failures</li> </ul>"},{"location":"changelog/#0210-2022-07-03","title":"[0.2.10] - 2022-07-03","text":"<ul> <li>Fix pip instalation is missing setuptools</li> <li>Update mkdocs</li> </ul>"},{"location":"changelog/#029-2022-07-03","title":"[0.2.9] - 2022-07-03","text":"<ul> <li>Added iscc_compare function</li> <li>Optimized soft_hash_audio performance</li> <li>Removed Cython from build requirements</li> <li>Fixed api listing tool</li> <li>Updated codec architecture figure</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#028-2022-04-21","title":"[0.2.8] - 2022-04-21","text":"<ul> <li>Fixed bug with subtype for semantic code</li> <li>Changed URI representation to upper case</li> <li>Changed to disallow ISCC-ID creation from ISCC-IDs</li> <li>Added line conversion tool</li> <li>Removed source wheel distribution</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#027-2022-04-16","title":"[0.2.7] - 2022-04-16","text":"<ul> <li>Fixed bug in iscc_id_incr_v0</li> <li>Added support to accept ISCC-ID URI as input for iscc_id_incr_v0</li> <li>Added guard against custom subtype in random ISCC-CODE generation.</li> </ul>"},{"location":"changelog/#026-2022-04-13","title":"[0.2.6] - 2022-04-13","text":"<ul> <li>Added <code>KY</code> and <code>MM</code> to valid prefixes</li> <li>Added support to check for compiled extension modules</li> <li>Added universal wheel distribution</li> </ul>"},{"location":"changelog/#025-2022-04-10","title":"[0.2.5] - 2022-04-10","text":"<ul> <li>Fixed missing <code>jcs</code> dependency</li> <li>Added SubType <code>NONE</code> to MT.ISCC to distinquish from SUM</li> <li>Added support for deterministic generation of random ISCC-CODEs</li> <li>Added support for custom bit-sizes for random ISCC-CODEs</li> <li>Moved changelog into separate file</li> <li>Updated dependencies</li> </ul>"},{"location":"changelog/#024-2022-03-19","title":"[0.2.4] - 2022-03-19","text":"<ul> <li>Updated dependencies</li> <li>Added Flake.from_int and Flake.from_string</li> <li>Made Flake comparable and hashable</li> <li>Use standard hex encoded multihash for datahash and metahash</li> </ul>"},{"location":"changelog/#023-2022-03-06","title":"[0.2.3] - 2022-03-06","text":"<ul> <li>Update to iscc-schema 0.3.3</li> <li>Change image normalization instructions</li> <li>Fix issue with exporting cdc cython only functions</li> </ul>"},{"location":"changelog/#021-2022-03-03","title":"[0.2.1] - 2022-03-03","text":"<ul> <li>Cleanup and update dependencies</li> <li>Fix bitarray api change</li> <li>Fix developer commands</li> </ul>"},{"location":"changelog/#020-2022-02-24","title":"[0.2.0] - 2022-02-24","text":"<ul> <li>Complete API refactoring</li> <li>Use Data-URL as input for Meta-Code</li> <li>Use wallet address for ISCC-ID creation</li> <li>Added new Flake-Code (distributed time/random ID)</li> <li>Replaced assertions with exeptions</li> <li>Use secure random functions</li> <li>Retired Python 3.6 support (EOL)</li> <li>Return simple <code>dict</code> objects from generator functions</li> <li>Added ISCC string validation</li> <li>Added multiple helper functions</li> </ul>"},{"location":"changelog/#019-2021-12-17","title":"[0.1.9] - 2021-12-17","text":"<ul> <li>Added warning on non-standard options</li> <li>Added multiformats support</li> <li>Added uri representation</li> <li>Removed redundant cdc_avg_chunk_size option</li> <li>Updated codec format documentation</li> </ul>"},{"location":"changelog/#018-2021-12-12","title":"[0.1.8] - 2021-12-12","text":"<ul> <li>Added conformance tests for all top level functions</li> <li>Added conformance tests to source dir</li> <li>Added conformance module with <code>selftest</code> function</li> <li>Changed gen_image_code to accept normalized pixels instead of stream</li> <li>Changed opts to core_opts</li> <li>Removed image pre-processing and Pillow dependency</li> <li>Fixed readability of conformance tests</li> <li>Fixed soft_hash_video_v0 to accept non-tuple sequences</li> <li>Updated example code</li> </ul>"},{"location":"changelog/#017-2021-12-09","title":"[0.1.7] - 2021-12-09","text":"<ul> <li>Add dotenv for enviroment based configuration</li> <li>Cleanup package toplevel imports</li> <li>Return schema objects for iscc_code and iscc_id</li> <li>Exclude unset and none values from result dicts</li> <li>Add support for multiple code combinations for ISCC-CODE</li> <li>Add support for ISCC-ID based on singular Instance-Code</li> <li>Add initial conformance test system</li> </ul>"},{"location":"changelog/#016-2021-11-29","title":"[0.1.6] - 2021-11-29","text":"<ul> <li>Show counter for ISCC-ID in Code.explain</li> </ul>"},{"location":"changelog/#015-2021-11-28","title":"[0.1.5] - 2021-11-28","text":"<ul> <li>Fix documentation</li> <li>Change metahash creation logic</li> <li>Refactor models</li> <li>Add Content-Code-Mixed</li> <li>Add ISCC-ID</li> <li>Refactor <code>compose</code> to <code>gen_iscc_code</code></li> <li>Refactor <code>models</code> to <code>schema</code></li> </ul>"},{"location":"changelog/#014-2021-11-17","title":"[0.1.4] - 2021-11-17","text":"<ul> <li>Simplified options</li> <li>Optimize video WTA-hash for use with 64-bit granular features</li> </ul>"},{"location":"changelog/#013-2021-11-15","title":"[0.1.3] - 2021-11-15","text":"<ul> <li>Try to compile Cython/C accelerator modules when installing via pip</li> <li>Simplify soft_hash api return values</li> <li>Add .code() method to InstanceHasher, DataHasher</li> <li>Remove granular fingerprint calculation</li> <li>Add more top-level imports</li> </ul>"},{"location":"changelog/#012-2021-11-14","title":"[0.1.2] - 2021-11-14","text":"<ul> <li>Export more functions to toplevel</li> <li>Return schema driven objects from ISCC code generators.</li> </ul>"},{"location":"changelog/#011-2021-11-14","title":"[0.1.1] - 2021-11-14","text":"<ul> <li>Fix packaging problems</li> </ul>"},{"location":"changelog/#010-2021-11-13","title":"[0.1.0] - 2021-11-13","text":"<ul> <li>Initial release</li> </ul>"},{"location":"conformance/","title":"ISCC - Conformance Testing","text":"<p>An application that claims ISCC conformance MUST pass all core functions from the ISCC conformance test suite. The test suite is available as JSON data on GitHub. Test data is structured as follows:</p> <pre><code>{\n\"&lt;function_name&gt;\": {\n\"&lt;test_name&gt;\": {\n\"inputs\": [\"&lt;value1&gt;\", \"&lt;value2&gt;\"],\n\"outputs\": [\"value1&gt;\", \"&lt;value2&gt;\"]\n}\n}\n}\n</code></pre> <p>Inputs that are expected to be <code>raw bytes or byte-streams</code> are embedded as HEX encoded strings in JSON and prefixed with <code>stream:</code> or <code>bytes</code> to support automated decoding during implementation testing.</p> <p>Example</p> <p>Byte-stream outputs in JSON test data: <pre><code>\"gen_data_code_v0\": {\n\"test_0000_two_bytes_64\": {\n\"inputs\": [\n\"stream:ff00\",\n64\n],\n\"outputs\": {\n\"iscc\": \"GAAXL2XYM5BQIAZ3\"\n}\n},\n...\n</code></pre></p>"},{"location":"conformance/#iscc_core.conformance.conformance_testdata","title":"<code>conformance_testdata()</code>","text":"<p>Yield tuples of test data.</p> <p>Returns:</p> Type Description <code>Generator[Tuple[str, Callable, List[Any], List[Any]]]</code> <p>Tuple with testdata (test_name, func_obj, inputs, outputs)</p>"},{"location":"conformance/#iscc_core.conformance.conformance_selftest","title":"<code>conformance_selftest()</code>","text":"<p>Run conformance tests.</p> <p>Returns:</p> Type Description <code>bool</code> <p>whether all tests passed</p>"},{"location":"constants/","title":"ISCC - Types and Constants","text":""},{"location":"constants/#iscc_core.constants.MT","title":"<code>MT</code>","text":""},{"location":"constants/#iscc_core.constants.MT--mt-maintypes","title":"MT - MainTypes","text":"Uint Symbol Bits Purpose 0 META 0000 Match on metadata similarity 1 SEMANTIC 0001 Match on semantic content similarity 2 CONTENT 0010 Match on perceptual content similarity 3 DATA 0011 Match on data similarity 4 INSTANCE 0100 Match on data identity 5 ISCC 0101 Composite of two or more ISCC-UNITs with common header"},{"location":"constants/#iscc_core.constants.ST","title":"<code>ST</code>","text":""},{"location":"constants/#iscc_core.constants.ST--st-subtypes","title":"ST - SubTypes","text":"Uint Symbol Bits Purpose 0 NONE 0000 For MainTypes that do not specify SubTypes"},{"location":"constants/#iscc_core.constants.ST_CC","title":"<code>ST_CC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_CC--st_cc","title":"ST_CC","text":"<p>SubTypes for <code>MT.CONTENT</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Match on syntactic text similarity 1 IMAGE 0001 Match on perceptual image similarity 2 AUDIO 0010 Match on audio chroma similarity 3 VIDEO 0011 Match on perceptual similarity 4 MIXED 0100 Match on similarity of content codes"},{"location":"constants/#iscc_core.constants.ST_ISCC","title":"<code>ST_ISCC</code>","text":""},{"location":"constants/#iscc_core.constants.ST_ISCC--st_iscc","title":"ST_ISCC","text":"<p>SubTypes for <code>MT.ISCC</code></p> Uint Symbol Bits Purpose 0 TEXT 0000 Composite ISCC inlcuding Text-Code 1 IMAGE 0001 Composite ISCC inlcuding Image-Code 2 AUDIO 0010 Composite ISCC inlcuding Audio-Code 3 VIDEO 0011 Composite ISCC inlcuding Video-Code 4 MIXED 0100 Composite ISCC inlcuding Mixed-Code 5 SUM 0101 Composite ISCC inlcuding only Data- and Instance-Code 6 NONE 0110 Composite ISCC including Meta, Data and Instance-Code"},{"location":"constants/#iscc_core.constants.VS","title":"<code>VS</code>","text":""},{"location":"constants/#iscc_core.constants.VS--vs-version","title":"VS - Version","text":"<p>Code Version</p> Uint Symbol Bits Purpose 0 V0 0000 Initial Version of Code without breaking changes"},{"location":"constants/#iscc_core.constants.LN","title":"<code>LN</code>","text":""},{"location":"constants/#iscc_core.constants.LN--ln-length","title":"LN - Length","text":"<p>Valid lengths for hash-digests.</p> <ul> <li>L32 = 32</li> <li>L64 = 64</li> <li>L72 = 72</li> <li>L80 = 80</li> <li>L96 = 96</li> <li>L128 = 128</li> <li>L160 = 160</li> <li>L192 = 192</li> <li>L224 = 224</li> <li>L256 = 256</li> </ul>"},{"location":"constants/#iscc_core.constants.MULTIBASE","title":"<code>MULTIBASE</code>","text":"<p>Supported Multibase encodings.</p> <ul> <li>base16 -&gt; f</li> <li>base32 -&gt; b</li> <li>base32hex -&gt; v</li> <li>base58btc -&gt; z</li> <li>base64url -&gt; u</li> </ul>"},{"location":"iscc_code/","title":"ISCC-CODE","text":"<p>A multi-component identifier for digital media assets.</p> <p>An ISCC-CODE can be generated from the concatenation of the digests of the following five ISCC-UNITs together with a single common header:</p> <ul> <li>Meta-Code - Encodes metadata similarity</li> <li>Semantic-Code - Encodes semantic content similarity (to be developed)</li> <li>Content-Code - Encodes syntactic/perceptual similarity</li> <li>Data-Code - Encodes raw bitstream similarity</li> <li>Instance-Code - Data checksum</li> </ul> <p>The following sequences of ISCC-UNITs are possible:</p> <ul> <li>Data, Instance</li> <li>Content, Data, Instance</li> <li>Semantic, Data, Instance</li> <li>Content, Semantic, Data, Instance</li> <li>Meta, Data, Instance</li> <li>Meta, Content, Data, Instance</li> <li>Meta, Semantic, Data, Instance</li> <li>Meta, Semantic, Content, Data, Instance</li> </ul>"},{"location":"iscc_code/#iscc_core.iscc_code.gen_iscc_code_v0","title":"<code>gen_iscc_code_v0(codes)</code>","text":"<p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iscc_id/","title":"ISCC-ID","text":"<p>A decentralized, owned, and short identifier for digital assets.</p> <p>The ISCC-ID is generated from a similarity-hash of the units of an ISCC-CODE together with a blockchain wallet address. Its SubType designates the blockchain from which the ISCC-ID was minted. The similarity-hash is always at least 64-bits and optionally suffixed with a <code>uvarint</code> endcoded <code>uniqueness counter</code>. The <code>uniqueness counter</code> is added and incremented only if the mint colides with a pre-existing ISCC-ID minted from the same blockchain from a different ISCC-CODE or from an identical ISCC-CODE registered by a different signatory.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id","title":"<code>gen_iscc_id(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate  ISCC-ID from ISCC-CODE with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.gen_iscc_id_v0","title":"<code>gen_iscc_id_v0(iscc_code, chain_id, wallet, uc = 0)</code>","text":"<p>Generate an ISCC-ID from an ISCC-CODE with uniqueness counter 'uc' with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>The ISCC-CODE from which to mint the ISCC-ID.</p> required <code>chain_id</code> <code>int</code> <p>Chain-ID of blockchain from which the ISCC-ID is minted.</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter of ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with an ISCC-ID</p>"},{"location":"iscc_id/#iscc_core.iscc_id.soft_hash_iscc_id_v0","title":"<code>soft_hash_iscc_id_v0(iscc_code, wallet, uc = 0)</code>","text":"<p>Calculate ISCC-ID hash digest from ISCC-CODE with algorithm v0.</p> <p>Accepts an ISCC-CODE or any sequence of ISCC-UNITs.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>ISCC-CODE</p> required <code>wallet</code> <code>str</code> <p>The wallet address that signes the ISCC declaration</p> required <code>uc</code> <code>int</code> <p>Uniqueness counter for ISCC-ID.</p> <code>0</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Digest for ISCC-ID without header but including uniqueness counter.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr","title":"<code>iscc_id_incr(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one.</p>"},{"location":"iscc_id/#iscc_core.iscc_id.iscc_id_incr_v0","title":"<code>iscc_id_incr_v0(iscc_id)</code>","text":"<p>Increment uniqueness counter of an ISCC-ID with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>iscc_id</code> <code>str</code> <p>Base32-encoded ISCC-ID.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32-encoded ISCC-ID with counter incremented by one (without \"ISCC:\" prefix).</p>"},{"location":"iscc_id/#iscc_core.iscc_id.alg_simhash_from_iscc_id","title":"<code>alg_simhash_from_iscc_id(iscc_id, wallet)</code>","text":"<p>Extract similarity preserving hex-encoded hash digest from ISCC-ID</p> <p>We need to un-xor the ISCC-ID hash digest with the wallet address hash to obtain the similarity preserving bytestring.</p>"},{"location":"iso-reference/","title":"ISCC - ISO Reference","text":"<p>The following functions are the reference implementations of ISO 24138:</p>"},{"location":"iso-reference/#iso-24138-51-meta-code","title":"ISO 24138 / 5.1 Meta-Code","text":"<code>gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits)</code> # <p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"iso-reference/#iso-24138-53-text-code","title":"ISO 24138 / 5.3 Text-Code","text":"<code>gen_text_code_v0(text, bits=ic.core_opts.text_bits)</code> # <p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"iso-reference/#iso-24138-54-image-code","title":"ISO 24138 / 5.4 Image-Code","text":"<code>gen_image_code_v0(pixels, bits=ic.core_opts.image_bits)</code> # <p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p> Source code in <code>iscc_core\\code_content_image.py</code> <pre><code>def gen_image_code_v0(pixels, bits=ic.core_opts.image_bits):\n# type: (Sequence[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Image with algorithm v0.\n    :param Sequence[int] pixels: Normalized image pixels (32x32 flattened gray values)\n    :param int bits: Bit-length of ISCC Content-Code Image (default 64).\n    :return: ISCC object with Content-Code Image.\n    :rtype: ISCC\n    \"\"\"\ndigest = soft_hash_image_v0(pixels, bits=bits)\nimage_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.IMAGE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + image_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-55-audio-code","title":"ISO 24138 / 5.5 Audio-Code","text":"<code>gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits)</code> # <p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p> Source code in <code>iscc_core\\code_content_audio.py</code> <pre><code>def gen_audio_code_v0(cv, bits=ic.core_opts.audio_bits):\n# type: (Iterable[int], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code Audio with algorithm v0.\n    :param Iterable[int] cv: Chromaprint vector\n    :param int bits: Bit-length resulting Content-Code Audio (multiple of 64)\n    :return: ISCC object with Content-Code Audio\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_audio_v0(cv, bits=bits)\naudio_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.AUDIO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + audio_code\nreturn {\"iscc\": iscc}\n</code></pre>"},{"location":"iso-reference/#iso-24138-56-video-code","title":"ISO 24138 / 5.6 Video-Code","text":"<code>gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits)</code> # <p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p> Source code in <code>iscc_core\\code_content_video.py</code> <pre><code>def gen_video_code_v0(frame_sigs, bits=ic.core_opts.video_bits):\n# type: (Sequence[ic.FrameSig], int) -&gt; dict\n\"\"\"\n    Create an ISCC Video-Code with algorithm v0.\n    :param ic.FrameSig frame_sigs: Sequence of MP7 frame signatures\n    :param int bits: Bit-length resulting Video-Code (multiple of 64)\n    :return: ISCC object with Video-Code\n    :rtype: dict\n    \"\"\"\ndigest = soft_hash_video_v0(frame_sigs, bits=bits)\nvideo_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.VIDEO,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + video_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-57-mixed-code","title":"ISO 24138 / 5.7 Mixed-Code","text":"<code>gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits)</code> # <p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p> Source code in <code>iscc_core\\code_content_mixed.py</code> <pre><code>def gen_mixed_code_v0(codes, bits=ic.core_opts.mixed_bits):\n# type: (Sequence[str], int) -&gt; dict\n\"\"\"\n    Create an ISCC Content-Code-Mixed with algorithm v0.\n    If the provided codes are of mixed length they are stripped to `bits` length for\n    calculation.\n    :param Iterable[str] codes: a list of Content-Codes.\n    :param int bits: Target bit-length of generated Content-Code-Mixed.\n    :return: ISCC object with Content-Code Mixed.\n    :rtype: dict\n    \"\"\"\ndigests = [ic.decode_base32(ic.iscc_clean(code)) for code in codes]\ndigest = soft_hash_codes_v0(digests, bits=bits)\nmixed_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.MIXED,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + mixed_code\nreturn dict(iscc=iscc, parts=list(codes))\n</code></pre>"},{"location":"iso-reference/#iso-24138-58-data-code","title":"ISO 24138 / 5.8 Data-Code","text":"<code>gen_data_code_v0(stream, bits=ic.core_opts.data_bits)</code> # <p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p> Source code in <code>iscc_core\\code_data.py</code> <pre><code>def gen_data_code_v0(stream, bits=ic.core_opts.data_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Data-Code with algorithm v0.\n    :param Stream stream: Input data stream.\n    :param int bits: Bit-length of ISCC Data-Code (default 64).\n    :return: ISCC object with Data-Code\n    :rtype: dict\n    \"\"\"\nhasher = DataHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ndata_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + data_code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"iso-reference/#iso-24138-59-instance-code","title":"ISO 24138 / 5.9 Instance-Code","text":"<code>gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits)</code> # <p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p> Source code in <code>iscc_core\\code_instance.py</code> <pre><code>def gen_instance_code_v0(stream, bits=ic.core_opts.instance_bits):\n# type: (ic.Stream, int) -&gt; dict\n\"\"\"\n    Create an ISCC Instance-Code with algorithm v0.\n    :param Stream stream: Binary data stream for Instance-Code generation\n    :param int bits: Bit-length of resulting Instance-Code (multiple of 64)\n    :return: ISCC object with Instance-Code and properties: datahash, filesize\n    :rtype: dict\n    \"\"\"\nhasher = InstanceHasherV0()\ndata = stream.read(ic.core_opts.io_read_size)\nwhile data:\nhasher.push(data)\ndata = stream.read(ic.core_opts.io_read_size)\ninstance_code = hasher.code(bits=bits)\niscc = \"ISCC:\" + instance_code\ninstance_code_obj = dict(\niscc=iscc,\ndatahash=hasher.multihash(),\nfilesize=hasher.filesize,\n)\nreturn instance_code_obj\n</code></pre>"},{"location":"iso-reference/#iso-24138-60-iscc-code","title":"ISO 24138 / 6.0 ISCC-CODE","text":"<code>gen_iscc_code_v0(codes)</code> # <p>Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Sequence[str]</code> <p>A valid sequence of singluar ISCC-UNITS.</p> required <p>Returns:</p> Type Description <code>dict</code> <p>An ISCC object with ISCC-CODE</p> Source code in <code>iscc_core\\iscc_code.py</code> <pre><code>def gen_iscc_code_v0(codes):\n# type: (Sequence[str]) -&gt; dict\n\"\"\"\n    Combine multiple ISCC-UNITS to an ISCC-CODE with a common header using\n    algorithm v0.\n    :param Sequence[str] codes: A valid sequence of singluar ISCC-UNITS.\n    :return: An ISCC object with ISCC-CODE\n    :rtype: dict\n    \"\"\"\ncodes = [ic.iscc_clean(code) for code in codes]\n# Check basic constraints\nif len(codes) &lt; 2:\nraise ValueError(\"Minimum two ISCC units required to generate valid ISCC-CODE\")\nfor code in codes:\nif len(code) &lt; 16:\nraise ValueError(f\"Cannot build ISCC-CODE from units shorter than 64-bits: {code}\")\n# Decode units and sort by MainType\ndecoded = sorted(\n[ic.decode_header(ic.decode_base32(code)) for code in codes], key=itemgetter(0)\n)\nmain_types = tuple(d[0] for d in decoded)\nif main_types[-2:] != (ic.MT.DATA, ic.MT.INSTANCE):\nraise ValueError(f\"ISCC-CODE requires at least MT.DATA and MT.INSTANCE units.\")\n# Determine SubType (generic mediatype)\nsub_types = [t[1] for t in decoded if t[0] in {ic.MT.SEMANTIC, ic.MT.CONTENT}]\nif len(set(sub_types)) &gt; 1:\nraise ValueError(f\"Semantic-Code and Content-Code must be of same SubType\")\nst = sub_types.pop() if sub_types else ic.ST_ISCC.SUM if len(codes) == 2 else ic.ST_ISCC.NONE\n# Encode unit combination\nencoded_length = ic.encode_units(main_types[:-2])\n# Collect and truncate unit digests to 64-bit\ndigest = b\"\".join([t[-1][:8] for t in decoded])\nheader = ic.encode_header(ic.MT.ISCC, st, ic.VS.V0, encoded_length)\ncode = ic.encode_base32(header + digest)\niscc = \"ISCC:\" + code\nreturn dict(iscc=iscc)\n</code></pre>"},{"location":"algorithms/cdc/","title":"ISCC - Content Defined Chunking","text":"<p>Compatible with fastcdc</p>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_chunks","title":"<code>alg_cdc_chunks(data, utf32, avg_chunk_size = ic.core_opts.data_avg_chunk_size)</code>","text":"<p>A generator that yields data-dependent chunks for <code>data</code>.</p> <p>Usage Example:</p> <pre><code>for chunk in cdc_data_chunks(data):\nhash(chunk)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>Raw data for variable sized chunking.</p> required <code>utf32</code> <code>bool</code> <p>If true assume we are chunking text that is utf32 encoded.</p> required <code>avg_chunk_size</code> <code>int</code> <p>Target chunk size in number of bytes.</p> <code>ic.core_opts.data_avg_chunk_size</code> <p>Returns:</p> Type Description <code>Generator[bytes]</code> <p>A generator that yields data chunks of variable sizes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_chunks(data, utf32, avg_chunk_size=ic.core_opts.data_avg_chunk_size):\n# type: (Data, bool, int) -&gt; Generator[bytes, None, None]\n\"\"\"\n    A generator that yields data-dependent chunks for `data`.\n    Usage Example:\n    ```python\n    for chunk in cdc_data_chunks(data):\n        hash(chunk)\n    ```\n    :param bytes data: Raw data for variable sized chunking.\n    :param bool utf32: If true assume we are chunking text that is utf32 encoded.\n    :param int avg_chunk_size: Target chunk size in number of bytes.\n    :return: A generator that yields data chunks of variable sizes.\n    :rtype: Generator[bytes]\n    \"\"\"\nstream = io.BytesIO(data)\nbuffer = stream.read(ic.core_opts.io_read_size)\nif not buffer:\nyield b\"\"\nmi, ma, cs, mask_s, mask_l = alg_cdc_params(avg_chunk_size)\nbuffer = memoryview(buffer)\nwhile buffer:\nif len(buffer) &lt;= ma:\nbuffer = memoryview(bytes(buffer) + stream.read(ic.core_opts.io_read_size))\ncut_point = alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)\n# Make sure cut points are at 4-byte aligned for utf32 encoded text\nif utf32:\ncut_point -= cut_point % 4\nyield bytes(buffer[:cut_point])\nbuffer = buffer[cut_point:]\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_offset","title":"<code>alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l)</code>","text":"<p>Find breakpoint offset for a given buffer.</p> <p>Parameters:</p> Name Type Description Default <code>buffer</code> <code>Data</code> <p>The data to be chunked.</p> required <code>mi</code> <code>int</code> <p>Minimum chunk size.</p> required <code>ma</code> <code>int</code> <p>Maximung chunk size.</p> required <code>cs</code> <code>int</code> <p>Center size.</p> required <code>mask_s</code> <code>int</code> <p>Small mask.</p> required <code>mask_l</code> <code>int</code> <p>Large mask.</p> required <p>Returns:</p> Type Description <code>int</code> <p>Offset of dynamic cutpoint in number of bytes.</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_offset(buffer, mi, ma, cs, mask_s, mask_l):\n# type: (ic.Data, int, int, int, int, int) -&gt; int\n\"\"\"\n    Find breakpoint offset for a given buffer.\n    :param Data buffer: The data to be chunked.\n    :param int mi: Minimum chunk size.\n    :param int ma: Maximung chunk size.\n    :param int cs: Center size.\n    :param int mask_s: Small mask.\n    :param int mask_l: Large mask.\n    :return: Offset of dynamic cutpoint in number of bytes.\n    :rtype: int\n    \"\"\"\npattern = 0\ni = mi\nsize = len(buffer)\nbarrier = min(cs, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_s:\nreturn i + 1\ni += 1\nbarrier = min(ma, size)\nwhile i &lt; barrier:\npattern = (pattern &gt;&gt; 1) + ic.core_opts.cdc_gear[buffer[i]]\nif not pattern &amp; mask_l:\nreturn i + 1\ni += 1\nreturn i\n</code></pre>"},{"location":"algorithms/cdc/#iscc_core.cdc.alg_cdc_params","title":"<code>alg_cdc_params(avg_size: int) -&gt; tuple</code>","text":"<p>Calculate CDC parameters</p> <p>Parameters:</p> Name Type Description Default <code>avg_size</code> <code>int</code> <p>Target average size of chunks in number of bytes.</p> required <p>Returns:</p> Type Description <code>tuple</code> <p>Tuple of (min_size, max_size, center_size, mask_s, mask_l).</p> Source code in <code>iscc_core\\cdc.py</code> <pre><code>def alg_cdc_params(avg_size: int) -&gt; tuple:\n\"\"\"\n    Calculate CDC parameters\n    :param int avg_size: Target average size of chunks in number of bytes.\n    :returns: Tuple of (min_size, max_size, center_size, mask_s, mask_l).\n    \"\"\"\nceil_div = lambda x, y: (x + y - 1) // y\nmask = lambda b: 2**b - 1\nmin_size = avg_size // 4\nmax_size = avg_size * 8\noffset = min_size + ceil_div(min_size, 2)\ncenter_size = avg_size - offset\nbits = round(log2(avg_size))\nmask_s = mask(bits + 1)\nmask_l = mask(bits - 1)\nreturn min_size, max_size, center_size, mask_s, mask_l\n</code></pre>"},{"location":"algorithms/dct/","title":"ISCC - Discrete Cosine Transform","text":""},{"location":"algorithms/dct/#iscc_core.dct.alg_dct","title":"<code>alg_dct(v)</code>","text":"<p>Discrete cosine transform.</p> <p>See: nayuki.io.</p> <p>Parameters:</p> Name Type Description Default <code>v</code> <code>Sequence[float]</code> <p>Input vector for DCT calculation.</p> required <p>Returns:</p> Type Description <code>List</code> <p>DCT Transformed vector.</p> Source code in <code>iscc_core\\dct.py</code> <pre><code>def alg_dct(v):\n# type: (Sequence[float]) -&gt; List\n\"\"\"\n    Discrete cosine transform.\n    See: [nayuki.io](https://www.nayuki.io/page/fast-discrete-cosine-transform-algorithms).\n    :param Sequence[float] v: Input vector for DCT calculation.\n    :return: DCT Transformed vector.\n    :rtype: List\n    \"\"\"\nn = len(v)\nif n == 1:\nreturn list(v)\nelif n == 0 or n % 2 != 0:\nraise ValueError()\nelse:\nhalf = n // 2\nalpha = [(v[i] + v[-(i + 1)]) for i in range(half)]\nbeta = [\n(v[i] - v[-(i + 1)]) / (math.cos((i + 0.5) * math.pi / n) * 2.0) for i in range(half)\n]\nalpha = alg_dct(alpha)\nbeta = alg_dct(beta)\nresult = []\nfor i in range(half - 1):\nresult.append(alpha[i])\nresult.append(beta[i] + beta[i + 1])\nresult.append(alpha[-1])\nresult.append(beta[-1])\nreturn result\n</code></pre>"},{"location":"algorithms/minhash/","title":"ISCC - Minhash","text":""},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash","title":"<code>alg_minhash(features)</code>","text":"<p>Calculate a 64 dimensional minhash integer vector.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>List[int]</code> <p>Minhash vector</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash(features):\n# type: (List[int]) -&gt; List[int]\n\"\"\"\n    Calculate a 64 dimensional minhash integer vector.\n    :param List[int] features: List of integer features\n    :return: Minhash vector\n    :rtype: List[int]\n    \"\"\"\nreturn [\nmin([(((a * f + b) &amp; MAXI64) % MPRIME) &amp; MAXH for f in features]) for a, b in zip(MPA, MPB)\n]\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_64","title":"<code>alg_minhash_64(features)</code>","text":"<p>Create 64-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>64-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_64(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 64-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 64-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 1)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_256","title":"<code>alg_minhash_256(features)</code>","text":"<p>Create 256-bit minimum hash digest.</p> <p>Parameters:</p> Name Type Description Default <code>features</code> <code>List[int]</code> <p>List of integer features</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_256(features):\n# type: (List[int]) -&gt; bytes\n\"\"\"\n    Create 256-bit minimum hash digest.\n    :param List[int] features: List of integer features\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nreturn alg_minhash_compress(alg_minhash(features), 4)\n</code></pre>"},{"location":"algorithms/minhash/#iscc_core.minhash.alg_minhash_compress","title":"<code>alg_minhash_compress(mhash, lsb = 4)</code>","text":"<p>Compress minhash vector to byte hash-digest.</p> <p>Concatenates <code>lsb</code> number of least-significant bits from each integer in <code>mhash</code>. For example an <code>mhash</code> with 64 integers and <code>lsb=4</code> will produce a 256-bit summary of the minhash vector.</p> <p>Parameters:</p> Name Type Description Default <code>mhash</code> <code>List[int]</code> <p>List of minhash integer features</p> required <code>lsb</code> <code>int</code> <p>Number of the least significant bits to retain</p> <code>4</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit binary from the least significant bits of the minhash values</p> Source code in <code>iscc_core\\minhash.py</code> <pre><code>def alg_minhash_compress(mhash, lsb=4):\n# type: (List[int], int) -&gt; bytes\n\"\"\"\n    Compress minhash vector to byte hash-digest.\n    Concatenates `lsb` number of least-significant bits from each integer in `mhash`.\n    For example an `mhash` with 64 integers and `lsb=4` will produce a 256-bit summary\n    of the minhash vector.\n    :param List[int] mhash: List of minhash integer features\n    :param int lsb: Number of the least significant bits to retain\n    :return: 256-bit binary from the least significant bits of the minhash values\n    :rtype: bytes\n    \"\"\"\nbits: str = \"\"\nfor bitpos in range(lsb):\nfor h in mhash:\nbits += str(h &gt;&gt; bitpos &amp; 1)\nreturn int(bits, 2).to_bytes((len(bits) + 7) // 8, \"big\")\n</code></pre>"},{"location":"algorithms/simhash/","title":"ISCC - Simhash","text":""},{"location":"algorithms/simhash/#iscc_core.simhash.alg_simhash","title":"<code>alg_simhash(hash_digests)</code>","text":"<p>Creates a similarity preserving hash from a sequence of equal sized hash digests.</p> <p>Parameters:</p> Name Type Description Default <code>hash_digests</code> <code>list</code> <p>A sequence of equaly sized byte-hashes.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity byte-hash</p> Source code in <code>iscc_core\\simhash.py</code> <pre><code>def alg_simhash(hash_digests):\n# type: (list[bytes]) -&gt; bytes\n\"\"\"\n    Creates a similarity preserving hash from a sequence of equal sized hash digests.\n    :param list hash_digests: A sequence of equaly sized byte-hashes.\n    :returns: Similarity byte-hash\n    :rtype: bytes\n    \"\"\"\nn_bytes = len(hash_digests[0])\nn_bits = n_bytes * 8\nvector = [0] * n_bits\nfor digest in hash_digests:\nh = bitarray()\nh.frombytes(digest)\nfor i in range(n_bits):\nvector[i] += h[i]\nminfeatures = len(hash_digests) / 2\nshash = 0\nfor i in range(n_bits):\nif vector[i] &gt;= minfeatures:\nshash |= 1 &lt;&lt; (n_bits - 1 - i)\nreturn shash.to_bytes(n_bytes, \"big\")\n</code></pre>"},{"location":"algorithms/wtahash/","title":"ISCC - Winner Takes All Hash","text":""},{"location":"algorithms/wtahash/#iscc_core.wtahash.alg_wtahash","title":"<code>alg_wtahash(vec: Sequence[float], bits: Sequence[float]) -&gt; bytes</code>","text":"<p>Calculate WTA Hash for vector with 380 values (MP7 frame signature).</p> Source code in <code>iscc_core\\wtahash.py</code> <pre><code>def alg_wtahash(vec: Sequence[float], bits) -&gt; bytes:\n\"\"\"Calculate WTA Hash for vector with 380 values (MP7 frame signature).\"\"\"\nh = []\nfor perm in WTA_VIDEO_ID_PERMUTATIONS:\nv = vec[perm[0]], vec[perm[1]]\nh.append(v.index(max(v)))\nif len(h) == bits:\nbreak\nh = bitarray(h).tobytes()\nreturn h\n</code></pre>"},{"location":"codec/","title":"ISCC - Codec","text":"<p>This module implements encoding, decoding and transcoding functions of ISCC</p>"},{"location":"codec/#codec-overview","title":"Codec Overview","text":""},{"location":"codec/#codec-functions","title":"Codec Functions","text":""},{"location":"codec/#iscc_core.codec.encode_component","title":"<code>encode_component(mtype, stype, version, bit_length, digest)</code>","text":"<p>Encode an ISCC-UNIT inlcuding header and body with standard base32 encoding.</p> <p>Note</p> <p>The <code>length</code> value must be the length in number of bits for the component. If <code>digest</code> has more bits than specified by <code>length</code> it wil be truncated.</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>Maintype of unit (0-6)</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit depending on MainType (0-5)</p> required <code>version</code> <code>Version</code> <p>Version of unit algorithm (0).</p> required <code>bit_length</code> <code>length</code> <p>Length of unit, in number of bits (multiple of 32)</p> required <code>digest</code> <code>bytes</code> <p>The hash digest of the unit.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Base32 encoded ISCC-UNIT.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_component(mtype, stype, version, bit_length, digest):\n# type: (MainType, SubType, Version, Length, bytes) -&gt; str\n\"\"\"\n    Encode an ISCC-UNIT inlcuding header and body with standard base32 encoding.\n    !!! note\n        The `length` value must be the **length in number of bits** for the component.\n        If `digest` has more bits than specified by `length` it wil be truncated.\n    :param MainType mtype: Maintype of unit (0-6)\n    :param SubType stype: SubType of unit depending on MainType (0-5)\n    :param Version version: Version of unit algorithm (0).\n    :param length bit_length: Length of unit, in number of bits (multiple of 32)\n    :param bytes digest: The hash digest of the unit.\n    :return: Base32 encoded ISCC-UNIT.\n    :rtype: str\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.ID, MT.FLAKE):\nencoded_length = encode_length(mtype, bit_length)\nelif mtype == MT.ISCC:\nraise ValueError(f\"{mtype} is not a unit\")\nelse:\nraise ValueError(f\"Illegal MainType {mtype}\")\nnbytes = bit_length // 8\nheader = encode_header(mtype, stype, version, encoded_length)\nbody = digest[:nbytes]\ncomponent_code = encode_base32(header + body)\nreturn component_code\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_header","title":"<code>encode_header(mtype, stype, version = 0, length = 1)</code>","text":"<p>Encodes header values with nibble-sized (4-bit) variable-length encoding. The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles is uneven it is padded with 4-bit <code>0000</code> at the end.</p> <p>Warning</p> <p>The length value must be encoded beforhand because its semantics depend on the MainType (see <code>encode_length</code> function).</p> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>MainType of unit.</p> required <code>stype</code> <code>SubType</code> <p>SubType of unit.</p> required <code>version</code> <code>Version</code> <p>Version of component algorithm.</p> <code>0</code> <code>length</code> <code>Length</code> <p>length value of unit (1 means 64-bits for standard units)</p> <code>1</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Varnibble stream encoded ISCC header as bytes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_header(mtype, stype, version=0, length=1):\n# type: (MainType, SubType, Version, Length) -&gt; bytes\n\"\"\"\n    Encodes header values with nibble-sized (4-bit) variable-length encoding.\n    The result is minimum 2 and maximum 8 bytes long. If the final count of nibbles\n    is uneven it is padded with 4-bit `0000` at the end.\n    !!! warning\n        The length value must be encoded beforhand because its semantics depend on\n        the MainType (see `encode_length` function).\n    :param MainType mtype: MainType of unit.\n    :param SubType stype: SubType of unit.\n    :param Version version: Version of component algorithm.\n    :param Length length: length value of unit (1 means 64-bits for standard units)\n    :return: Varnibble stream encoded ISCC header as bytes.\n    :rtype: bytes\n    \"\"\"\nheader = bitarray()\nfor n in (mtype, stype, version, length):\nheader += encode_varnibble(n)\n# Append zero-padding if required (right side, least-significant bits).\nheader.fill()\nreturn header.tobytes()\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_header","title":"<code>decode_header(data)</code>","text":"<p>Decodes varnibble encoded header and returns it together with <code>tail data</code>.</p> <p>Tail data is included to enable decoding of sequential ISCCs. The returned tail data must be truncated to decode_length(r[0], r[3]) bits to recover the actual hash-bytes.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>bytes</code> <p>ISCC bytes</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>(MainType, SubType, Version, length, TailData)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_header(data):\n# type: (bytes) -&gt; IsccTuple\n\"\"\"\n    Decodes varnibble encoded header and returns it together with `tail data`.\n    Tail data is included to enable decoding of sequential ISCCs. The returned tail\n    data must be truncated to decode_length(r[0], r[3]) bits to recover the actual\n    hash-bytes.\n    :param bytes data: ISCC bytes\n    :return: (MainType, SubType, Version, length, TailData)\n    :rtype: IsccTuple\n    \"\"\"\nresult = []\nba = bitarray()\nba.frombytes(data)\ndata = ba\nfor _ in range(4):\nvalue, data = decode_varnibble(data)\nresult.append(value)\n# Strip 4-bit padding if required\nif len(data) % 8 and data[:4] == bitarray(\"0000\"):\ndata = data[4:]\nresult.append(data.tobytes())\nreturn tuple(result)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_varnibble","title":"<code>encode_varnibble(n)</code>","text":"<p>Writes integer to variable length sequence of 4-bit chunks.</p> <p>Variable-length encoding scheme:</p> prefix bits nibbles data bits unsigned range 0 1 3 0 - 7 10 2 6 8 - 71 110 3 9 72 - 583 1110 4 12 584 - 4679 <p>Parameters:</p> Name Type Description Default <code>n</code> <code>int</code> <p>Positive integer to be encoded as varnibble (0-4679)</p> required <p>Returns:</p> Type Description <code>bitarray</code> <p>Varnibble encoded integera</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_varnibble(n):\n# type: (int) -&gt; bitarray\n\"\"\"\n    Writes integer to variable length sequence of 4-bit chunks.\n    Variable-length encoding scheme:\n    ------------------------------------------------------\n    | prefix bits | nibbles | data bits | unsigned range |\n    | ----------- | ------- | --------- | -------------- |\n    | 0           | 1       | 3         | 0 - 7          |\n    | 10          | 2       | 6         | 8 - 71         |\n    | 110         | 3       | 9         | 72 - 583       |\n    | 1110        | 4       | 12        | 584 - 4679     |\n    :param int n: Positive integer to be encoded as varnibble (0-4679)\n    :return: Varnibble encoded integera\n    :rtype: bitarray\n    \"\"\"\nif 0 &lt;= n &lt; 8:\nreturn int2ba(n, length=4)\nelif 8 &lt;= n &lt; 72:\nreturn bitarray(\"10\") + int2ba(n - 8, length=6)\nelif 72 &lt;= n &lt; 584:\nreturn bitarray(\"110\") + int2ba(n - 72, length=9)\nelif 584 &lt;= n &lt; 4680:\nreturn bitarray(\"1110\") + int2ba(n - 584, length=12)\nelse:\nraise ValueError(\"Value must be between 0 and 4679\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_varnibble","title":"<code>decode_varnibble(b)</code>","text":"<p>Reads first varnibble, returns its integer value and remaining bits.</p> <p>Parameters:</p> Name Type Description Default <code>b</code> <code>bitarray</code> <p>Array of header bits</p> required <p>Returns:</p> Type Description <code>Tuple[int, bitarray]</code> <p>A tuple of the integer value of first varnible and the remaining bits.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_varnibble(b):\n# type: (bitarray) -&gt; Tuple[int, bitarray]\n\"\"\"Reads first varnibble, returns its integer value and remaining bits.\n    :param bitarray b: Array of header bits\n    :return: A tuple of the integer value of first varnible and the remaining bits.\n    :rtype: Tuple[int, bitarray]\n    \"\"\"\nbits = len(b)\nif bits &gt;= 4 and b[0] == 0:\nreturn ba2int(b[:4]), b[4:]\nif bits &gt;= 8 and b[1] == 0:\nreturn ba2int(b[2:8]) + 8, b[8:]\nif bits &gt;= 12 and b[2] == 0:\nreturn ba2int(b[3:12]) + 72, b[12:]\nif bits &gt;= 16 and b[3] == 0:\nreturn ba2int(b[4:16]) + 584, b[16:]\nraise ValueError(\"Invalid bitarray\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_units","title":"<code>encode_units(units)</code>","text":"<p>Encodes a combination of ISCC units to an integer between 0-7 to be used as length value for the final encoding of MT.ISCC</p> <p>Parameters:</p> Name Type Description Default <code>units</code> <code>Tuple</code> <p>A tuple of a MainType combination (can be empty)</p> required <p>Returns:</p> Type Description <code>int</code> <p>Integer value to be used as length-value for header encoding</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_units(units):\n# type: (Tuple[MT, ...]) -&gt; int\n\"\"\"\n    Encodes a combination of ISCC units to an integer between 0-7 to be used as length\n    value for the final encoding of MT.ISCC\n    :param Tuple units: A tuple of a MainType combination (can be empty)\n    :return: Integer value to be used as length-value for header encoding\n    :rtype: int\n    \"\"\"\nreturn UNITS.index(units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_units","title":"<code>decode_units(unit_id)</code>","text":"<p>Decodes an ISCC header length value that has been encoded with a unit_id to an ordered tuple of MainTypes.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_units(unit_id):\n# type: (int) -&gt; Tuple[MT, ...]\n\"\"\"\n    Decodes an ISCC header length value that has been encoded with a unit_id to an\n    ordered tuple of MainTypes.\n    \"\"\"\nunits = sorted(UNITS[unit_id])\nreturn tuple(MT(u) for u in units)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_length","title":"<code>encode_length(mtype, length)</code>","text":"<p>Encode length to integer value for header encoding.</p> <p>The <code>length</code> value has MainType-specific semantics:</p> <p>For MainTypes <code>META</code>, <code>SEMANTIC</code>, <code>CONTENT</code>, <code>DATA</code>, <code>INSTANCE</code>:</p> <pre><code>Length means number of bits for the body.\nLength is encoded as the multiple of 32-bit chunks (0 being 32bits)\nExamples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n</code></pre> <p>For MainType <code>ISCC</code>:</p> <pre><code>MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\noptional. Length means the composition of optional 64-bit units included\nin the ISCC composite.\n\nExamples:\n    No optional units      -&gt; 0000 -&gt; 0\n    CONTENT                -&gt; 0001 -&gt; 1\n    SEMANTIC               -&gt; 0010 -&gt; 2\n    SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n    META                   -&gt; 0100 -&gt; 4\n    META, CONTENT          -&gt; 0101 -&gt; 5\n    ...\n</code></pre> <p>For MainType <code>ID</code>:</p> <pre><code>Lengths means number the number of bits for the body including the counter\nLength is encoded as number of bytes of the counter (64-bit body is implicit)\nExamples:\n    64 -&gt; 0 (No counter)\n    72 -&gt; 1 (One byte counter)\n    80 -&gt; 2 (Two byte counter)\n    ...\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>mtype</code> <code>MainType</code> <p>The MainType for which to encode the length value.</p> required <code>length</code> <code>Length</code> <p>The length expressed according to the semantics of the type</p> required <p>Returns:</p> Type Description <code>int</code> <p>The length value encoded as integer for use with write_header.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_length(mtype, length):\n# type: (MainType, Length) -&gt; int\n\"\"\"\n    Encode length to integer value for header encoding.\n    The `length` value has MainType-specific semantics:\n    For MainTypes `META`, `SEMANTIC`, `CONTENT`, `DATA`, `INSTANCE`:\n        Length means number of bits for the body.\n        Length is encoded as the multiple of 32-bit chunks (0 being 32bits)\n        Examples: 32 -&gt; 0, 64 -&gt; 1, 96 -&gt; 2 ...\n    For MainType `ISCC`:\n        MainTypes `DATA` and `INSTANCE` are mandatory for ISCC-CODEs, all others are\n        optional. Length means the composition of optional 64-bit units included\n        in the ISCC composite.\n        Examples:\n            No optional units      -&gt; 0000 -&gt; 0\n            CONTENT                -&gt; 0001 -&gt; 1\n            SEMANTIC               -&gt; 0010 -&gt; 2\n            SEMANTIC, CONTENT      -&gt; 0011 -&gt; 3\n            META                   -&gt; 0100 -&gt; 4\n            META, CONTENT          -&gt; 0101 -&gt; 5\n            ...\n    For MainType `ID`:\n        Lengths means number the number of bits for the body including the counter\n        Length is encoded as number of bytes of the counter (64-bit body is implicit)\n        Examples:\n            64 -&gt; 0 (No counter)\n            72 -&gt; 1 (One byte counter)\n            80 -&gt; 2 (Two byte counter)\n            ...\n    :param MainType mtype: The MainType for which to encode the length value.\n    :param Length length: The length expressed according to the semantics of the type\n    :return: The length value encoded as integer for use with write_header.\n    :rtype: int\n    \"\"\"\nerror = f\"Invalid length {length} for MainType {mtype}\"\n# standard case (length field denotes number of 32-bit chunks, 0 being 32-bits)\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nif length &gt;= 32 and not length % 32:\nreturn (length // 32) - 1\nraise ValueError(error)\n# flag type encoding of included components (pass through as encoded out-of-band)\nelif mtype == MT.ISCC:\nif 0 &lt;= length &lt;= 7:\nreturn length\nraise ValueError(error)\n# counter byte length encoding\nelif mtype == MT.ID:\nif 64 &lt;= length &lt;= 96:\nreturn (length - 64) // 8\nraise ValueError(error)\nelse:\nraise ValueError(error)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_length","title":"<code>decode_length(mtype, length)</code>","text":"<p>Dedoce raw length value from ISCC header to length of digest in number of bits.</p> <p>Decodes a raw header integer value in to its semantically meaningfull value (e.g. number of bits)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_length(mtype, length):\n# type: (MainType, Length) -&gt; LN\n\"\"\"\n    Dedoce raw length value from ISCC header to length of digest in number of bits.\n    Decodes a raw header integer value in to its semantically meaningfull value (e.g.\n    number of bits)\n    \"\"\"\nif mtype in (MT.META, MT.SEMANTIC, MT.CONTENT, MT.DATA, MT.INSTANCE, MT.FLAKE):\nreturn LN((length + 1) * 32)\nelif mtype == MT.ISCC:\nreturn LN(len(decode_units(length)) * 64 + 128)\nelif mtype == MT.ID:\nreturn LN(length * 8 + 64)\nelse:\nraise ValueError(f\"Invalid length {length} for MainType {mtype}\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32","title":"<code>encode_base32(data)</code>","text":"<p>Standard RFC4648 base32 encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base32 encoding without padding.\n    \"\"\"\nreturn b32encode(data).decode(\"ascii\").rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32","title":"<code>decode_base32(code)</code>","text":"<p>Standard RFC4648 base32 decoding without padding and with casefolding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base32 decoding without padding and with casefolding.\n    \"\"\"\n# python stdlib does not support base32 without padding, so we have to re-pad.\ncl = len(code)\npad_length = math.ceil(cl / 8) * 8 - cl\nreturn bytes(b32decode(code + \"=\" * pad_length, casefold=True))\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_decompose","title":"<code>iscc_decompose(iscc_code)</code>","text":"<p>Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.</p> <p>A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated by a hyphen.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decompose(iscc_code):\n# type: (str) -&gt; List[str]\n\"\"\"\n    Decompose a normalized ISCC-CODE or any valid ISCC sequence into a list of ISCC-UNITS.\n    A valid ISCC sequence is a string concatenation of ISCC-UNITS optionally seperated\n    by a hyphen.\n    \"\"\"\niscc_code = iscc_clean(iscc_code)\ncomponents = []\nraw_code = decode_base32(iscc_code)\nwhile raw_code:\nmt, st, vs, ln, body = decode_header(raw_code)\n# standard ISCC-UNIT with tail continuation\nif mt != MT.ISCC:\nln_bits = decode_length(mt, ln)\ncode = encode_component(mt, st, vs, ln_bits, body[: ln_bits // 8])\ncomponents.append(code)\nraw_code = body[ln_bits // 8 :]\ncontinue\n# ISCC-CODE\nmain_types = decode_units(ln)\n# rebuild dynamic units (META, SEMANTIC, CONTENT)\nfor idx, mtype in enumerate(main_types):\nstype = ST.NONE if mtype == MT.META else st\ncode = encode_component(mtype, stype, vs, 64, body[idx * 8 :])\ncomponents.append(code)\n# rebuild static units (DATA, INSTANCE)\ndata_code = encode_component(MT.DATA, ST.NONE, vs, 64, body[-16:-8])\ninstance_code = encode_component(MT.INSTANCE, ST.NONE, vs, 64, body[-8:])\ncomponents.extend([data_code, instance_code])\nbreak\nreturn components\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_normalize","title":"<code>iscc_normalize(iscc_code)</code>","text":"<p>Normalize an ISCC to its canonical form.</p> <p>The canonical form of an ISCC is its shortest base32 encoded representation prefixed with the string <code>ISCC:</code>.</p> <p>Possible valid inputs:</p> <pre><code>MEACB7X7777574L6\nISCC:MEACB7X7777574L6\nfcc010001657fe7cafe9791bb\niscc:maagztfqttvizpjr\nIscc:Maagztfqttvizpjr\n</code></pre> <p>Info</p> <p>A concatenated sequence of codes will be composed into a single ISCC of MainType <code>MT.ISCC</code> if possible.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core\n&gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>iscc_code</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Normalized ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_normalize(iscc_code):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize an ISCC to its canonical form.\n    The canonical form of an ISCC is its shortest base32 encoded representation\n    prefixed with the string `ISCC:`.\n    Possible valid inputs:\n        MEACB7X7777574L6\n        ISCC:MEACB7X7777574L6\n        fcc010001657fe7cafe9791bb\n        iscc:maagztfqttvizpjr\n        Iscc:Maagztfqttvizpjr\n    !!! info\n        A concatenated sequence of codes will be composed into a single ISCC of MainType\n        `MT.ISCC` if possible.\n    !!! example\n        ``` py\n        &gt;&gt;&gt; import iscc_core\n        &gt;&gt;&gt; iscc_core.iscc_normalize(\"GAAW2PRCRS5LNVZV-IAAUVACQKXE3V44W\")\n        'ISCC:KUAG2PRCRS5LNVZVJKAFAVOJXLZZM'\n        ```\n    :param str iscc_code: Any valid ISCC string\n    :return: Normalized ISCC\n    :rtype: str\n    \"\"\"\nfrom iscc_core.iscc_code import gen_iscc_code_v0\ndecoders = {\nMULTIBASE.base16.value: bytes.fromhex,  # f\nMULTIBASE.base32.value: decode_base32,  # b\nMULTIBASE.base32hex.value: decode_base32hex,  # v\nMULTIBASE.base58btc.value: base58.b58decode,  # z\nMULTIBASE.base64url.value: decode_base64,  # u\n}\n# Transcode to base32 if &lt;multibase&gt;&lt;multicodec&gt; encoded\nmultibase_prefix = iscc_code[0]\nif multibase_prefix in decoders.keys():\ndecoder = decoders[multibase_prefix]\ndecoded = decoder(iscc_code[1:])\nif not decoded.startswith(MC_PREFIX):\nraise ValueError(f\"Malformed multiformat codec: {decoded[:2]}\")\niscc_code = encode_base32(decoded[2:])\nelse:\nprefix = iscc_code.upper().replace(\"ISCC:\", \"\")[:2]\nif prefix not in PREFIXES:\nraise ValueError(f\"ISCC starts with invalid prefix {prefix}\")\ndecomposed = iscc_decompose(iscc_code)\nrecomposed = gen_iscc_code_v0(decomposed)[\"iscc\"] if len(decomposed) &gt;= 2 else decomposed[0]\nreturn f\"ISCC:{recomposed}\" if not recomposed.startswith(\"ISCC:\") else recomposed\n</code></pre>"},{"location":"codec/#alternate-encodings","title":"Alternate Encodings","text":""},{"location":"codec/#iscc_core.codec.encode_base64","title":"<code>encode_base64(data)</code>","text":"<p>Standard RFC4648 base64url encoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base64(data):\n# type: (bytes) -&gt; str\n\"\"\"\n    Standard RFC4648 base64url encoding without padding.\n    \"\"\"\ncode = urlsafe_b64encode(data).decode(\"ascii\")\nreturn code.rstrip(\"=\")\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base64","title":"<code>decode_base64(code)</code>","text":"<p>Standard RFC4648 base64url decoding without padding.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base64(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    Standard RFC4648 base64url decoding without padding.\n    \"\"\"\npadding = 4 - (len(code) % 4)\nstring = code + (\"=\" * padding)\nreturn urlsafe_b64decode(string)\n</code></pre>"},{"location":"codec/#iscc_core.codec.encode_base32hex","title":"<code>encode_base32hex(data)</code>","text":"<p>RFC4648 Base32hex encoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def encode_base32hex(data):\n# type: (bytes) -&gt;  str\n\"\"\"\n    RFC4648 Base32hex encoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\nb32 = encode_base32(data)\nreturn b32.translate(b32_to_hex)\n</code></pre>"},{"location":"codec/#iscc_core.codec.decode_base32hex","title":"<code>decode_base32hex(code)</code>","text":"<p>RFC4648 Base32hex decoding without padding</p> <p>see: https://tools.ietf.org/html/rfc4648#page-10</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def decode_base32hex(code):\n# type: (str) -&gt; bytes\n\"\"\"\n    RFC4648 Base32hex decoding without padding\n    see: https://tools.ietf.org/html/rfc4648#page-10\n    \"\"\"\n# Make sure we use upper-case version for translation\nb32 = code.upper().translate(hex_to_b32)\nreturn decode_base32(b32)\n</code></pre>"},{"location":"codec/#helper-functions","title":"Helper Functions","text":""},{"location":"codec/#iscc_core.codec.iscc_decode","title":"<code>iscc_decode(iscc)</code>","text":"<p>Decode ISCC to an IsccTuple</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>IsccTuple</code> <p>ISCC decoded to a tuple</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_decode(iscc):\n# type: (str) -&gt; IsccTuple\n\"\"\"\n    Decode ISCC to an IsccTuple\n    :param str iscc: ISCC string\n    :return: ISCC decoded to a tuple\n    :rtype: IsccTuple\n    \"\"\"\niscc = iscc_clean(iscc_normalize(iscc))\ndata = decode_base32(iscc)\nreturn decode_header(data)\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_explain","title":"<code>iscc_explain(iscc)</code>","text":"<p>Convert ISCC to a human-readable representation</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Human-readable representation of ISCC</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_explain(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Convert ISCC to a human-readable representation\n    :param str iscc: ISCC string\n    :return: Human-readable representation of ISCC\n    :rtype: str\n    \"\"\"\ntid = iscc_type_id(iscc)\nfields = iscc_decode(iscc)\nif fields[0] == MT.ID:\ncounter_bytes = fields[-1][8:]\nif counter_bytes:\ncounter = uvarint.decode(counter_bytes)\nhex_hash = fields[-1][:8].hex()\nreturn f\"{tid}-{hex_hash}-{counter.integer}\"\nhex_hash = fields[-1].hex()\nreturn f\"{tid}-{hex_hash}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_type_id","title":"<code>iscc_type_id(iscc)</code>","text":"<p>Extract and convert ISCC HEADER to a readable Type-ID string.</p> <p>Type-ids can be used as names in databases to index ISCC-UNITs seperatly.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Unique Type-ID string</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_type_id(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Extract and convert ISCC HEADER to a readable Type-ID string.\n    Type-ids can be used as names in databases to index ISCC-UNITs seperatly.\n    :param str iscc: ISCC string\n    :return: Unique Type-ID string\n    :rtype: str\n    \"\"\"\nfields = iscc_decode(iscc)\nmtype = MT(fields[0])\nstype = SUBTYPE_MAP[fields[0]](fields[1])\nif mtype == MT.ISCC:\nmtypes = decode_units(fields[3])\nlength = \"\".join([t.name[0] for t in mtypes]) + \"DI\"\nelse:\nlength = decode_length(fields[0], fields[3])\nversion = VS(fields[2])\nreturn f\"{mtype.name}-{stype.name}-{version.name}-{length}\"\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_validate","title":"<code>iscc_validate(iscc, strict = True)</code>","text":"<p>Validate that a given string is a strictly well-formed ISCC.</p> <p>A strictly well-formed ISCC is:</p> <ul> <li>an ISCC-CODE or ISCC-UNIT</li> <li>encoded with base32 upper without padding</li> <li>has a valid combination of header values</li> <li>is represented in its canonical form</li> </ul> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>ISCC string</p> required <code>strict</code> <code>bool</code> <p>Raise an exeption if validation fails (default True)</p> <code>True</code> <p>Returns:</p> Type Description <code>bool</code> <p>True if sting is valid else false. (raises ValueError in strict mode)</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_validate(iscc, strict=True):\n# type: (str, bool) -&gt; bool\n\"\"\"\n    Validate that a given string is a *strictly well-formed* ISCC.\n    A *strictly well-formed* ISCC is:\n    - an ISCC-CODE or ISCC-UNIT\n    - encoded with base32 upper without padding\n    - has a valid combination of header values\n    - is represented in its canonical form\n    :param str iscc: ISCC string\n    :param bool strict: Raise an exeption if validation fails (default True)\n    :return: True if sting is valid else false. (raises ValueError in strict mode)\n    :rtype: bool\n    \"\"\"\n# Basic regex validation\nmatch = CANONICAL_REGEX.match(iscc)\nif not match:\nif strict:\nraise ValueError(\"ISCC string does not match ^ISCC:[A-Z2-7]{10,68}$\")\nelse:\nreturn False\n# Base32 encoding test\ntry:\ndecode_base32(iscc.split(\":\")[1])\nexcept Exception as e:\nif strict:\nraise ValueError(e)\nelse:\nreturn False\ncleaned = iscc_clean(iscc)\n# Prefix test\nprefix = cleaned[:2]\nif prefix not in PREFIXES:\nif strict:\nraise ValueError(f\"Header starts with invalid sequence {prefix}\")\nelse:\nreturn False\n# Version test\nm, s, v, l, t = decode_header(decode_base32(cleaned))\nif v != 0:\nif strict:\nraise ValueError(f\"Unknown version {v} in version header\")\nelse:\nreturn False\n# Length test\nexpected_nbyptes = decode_length(m, l).value // 8\nactual_nbyptes = len(t)\nif expected_nbyptes != actual_nbyptes:\nif strict:\nraise ValueError(f\"Header expects {expected_nbyptes} but got {actual_nbyptes} bytes\")\nelse:\nreturn False\nreturn True\n</code></pre>"},{"location":"codec/#iscc_core.codec.iscc_clean","title":"<code>iscc_clean(iscc)</code>","text":"<p>Cleanup ISCC string.</p> <p>Removes leading scheme, dashes, leading/trailing whitespace.</p> <p>Parameters:</p> Name Type Description Default <code>iscc</code> <code>str</code> <p>Any valid ISCC string</p> required <p>Returns:</p> Type Description <code>str</code> <p>Cleaned ISCC string.</p> Source code in <code>iscc_core\\codec.py</code> <pre><code>def iscc_clean(iscc):\n# type: (str) -&gt; str\n\"\"\"\n    Cleanup ISCC string.\n    Removes leading scheme, dashes, leading/trailing whitespace.\n    :param str iscc: Any valid ISCC string\n    :return: Cleaned ISCC string.\n    :rtype: str\n    \"\"\"\nsplit = [part.strip() for part in iscc.strip().split(\":\")]\nif len(split) == 1:\ncode = split[0]\n# remove dashes if not multiformat\nif code[0] not in list(MULTIBASE):\ncode = code.replace(\"-\", \"\")\nreturn code\nelif len(split) == 2:\nscheme, code = split\nif scheme.lower() != \"iscc\":\nraise ValueError(f\"Invalid scheme: {scheme}\")\nreturn code.replace(\"-\", \"\")\nelse:\nraise ValueError(f\"Malformed ISCC string: {iscc}\")\n</code></pre>"},{"location":"options/options/","title":"ISCC-CORE - Configuration Options","text":"<p>Options for the iscc-core package can be configured using environment variables. Variables are loaded as class-attributes on the <code>CoreOptions</code> instance. Environment variables are named like the class-attribute but prefixed with <code>ISCC_CORE_</code> and upper-cased.</p> <p>Example how to access configuration options</p> <pre><code>import iscc_core as ic\n# To access ISCC_CORE_TEXT_NGRAM_SIZE setting use\ntext_ngram_size: int = ic.core_opts.text_ngram_size\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions","title":"CoreOptions","text":"<p>Parameters with defaults for ISCC calculations.</p>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_bits","title":"meta_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_bits: int = Field(\n64,\ndescription=\"Default length of generated Meta-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_name","title":"meta_trim_name  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_trim_name: int = Field(\n128, description=\"Trim `name` to this mumber of bytes\"\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_trim_description","title":"meta_trim_description  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_trim_description: int = Field(\n4096,\ndescription=\"Trim `description` to this number of bytes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_text","title":"meta_ngram_size_text  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_ngram_size_text: int = Field(\n3,\ndescription=\"Sliding window width (characters) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.meta_ngram_size_bytes","title":"meta_ngram_size_bytes  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>meta_ngram_size_bytes: int = Field(\n4,\ndescription=\"Sliding window width (bytes) for metadata\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_bits","title":"text_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Text in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_ngram_size","title":"text_ngram_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_ngram_size: int = Field(\n13,\ndescription=\"Number of characters per feature hash (size of sliding window)\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_unicode_filter","title":"text_unicode_filter  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_unicode_filter: frozenset = Field(\nfrozenset({\"C\", \"M\", \"P\"}),\ndescription=\"Unicode categories to remove during text normalization\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.text_newlines","title":"text_newlines  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>text_newlines: frozenset = Field(\nfrozenset(\n{\n\"\\n\",\n\"\\x0b\",\n\"\\x0c\",\n\"\\r\",\n\"\\x85\",\n\"\\u2028\",\n\"\\u2029\",\n}\n),\ndescription=\"Characters regarded as newline characters for normalization purposes\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.image_bits","title":"image_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>image_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Image in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.audio_bits","title":"audio_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>audio_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Audio in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.video_bits","title":"video_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>video_bits: int = Field(\n64,\ndescription=\"Default length of generated Content-Code Video in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_bits","title":"data_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>data_bits: int = Field(\n64,\ndescription=\"Default length of generated Data-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.data_avg_chunk_size","title":"data_avg_chunk_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>data_avg_chunk_size: int = Field(\n1024,\ndescription=\"Target chunk size for data chunking in number of bytes.\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.instance_bits","title":"instance_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>instance_bits: int = Field(\n64,\ndescription=\"Default length of generated Instance-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.mixed_bits","title":"mixed_bits  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>mixed_bits: int = Field(\n64,\ndescription=\"Default length of generated Mixed-Code in bits\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.io_read_size","title":"io_read_size  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>io_read_size: int = Field(\n2097152,\ndescription=\"File read buffer size in bytes for hashing operations\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.CoreOptions.cdc_gear","title":"cdc_gear  <code>instance-attribute</code> <code>class-attribute</code>","text":"<pre><code>cdc_gear: Tuple = Field(\n(\n1553318008,\n574654857,\n759734804,\n310648967,\n1393527547,\n1195718329,\n694400241,\n1154184075,\n1319583805,\n1298164590,\n122602963,\n989043992,\n1918895050,\n933636724,\n1369634190,\n1963341198,\n1565176104,\n1296753019,\n1105746212,\n1191982839,\n1195494369,\n29065008,\n1635524067,\n722221599,\n1355059059,\n564669751,\n1620421856,\n1100048288,\n1018120624,\n1087284781,\n1723604070,\n1415454125,\n737834957,\n1854265892,\n1605418437,\n1697446953,\n973791659,\n674750707,\n1669838606,\n320299026,\n1130545851,\n1725494449,\n939321396,\n748475270,\n554975894,\n1651665064,\n1695413559,\n671470969,\n992078781,\n1935142196,\n1062778243,\n1901125066,\n1935811166,\n1644847216,\n744420649,\n2068980838,\n1988851904,\n1263854878,\n1979320293,\n111370182,\n817303588,\n478553825,\n694867320,\n685227566,\n345022554,\n2095989693,\n1770739427,\n165413158,\n1322704750,\n46251975,\n710520147,\n700507188,\n2104251000,\n1350123687,\n1593227923,\n1756802846,\n1179873910,\n1629210470,\n358373501,\n807118919,\n751426983,\n172199468,\n174707988,\n1951167187,\n1328704411,\n2129871494,\n1242495143,\n1793093310,\n1721521010,\n306195915,\n1609230749,\n1992815783,\n1790818204,\n234528824,\n551692332,\n1930351755,\n110996527,\n378457918,\n638641695,\n743517326,\n368806918,\n1583529078,\n1767199029,\n182158924,\n1114175764,\n882553770,\n552467890,\n1366456705,\n934589400,\n1574008098,\n1798094820,\n1548210079,\n821697741,\n601807702,\n332526858,\n1693310695,\n136360183,\n1189114632,\n506273277,\n397438002,\n620771032,\n676183860,\n1747529440,\n909035644,\n142389739,\n1991534368,\n272707803,\n1905681287,\n1210958911,\n596176677,\n1380009185,\n1153270606,\n1150188963,\n1067903737,\n1020928348,\n978324723,\n962376754,\n1368724127,\n1133797255,\n1367747748,\n1458212849,\n537933020,\n1295159285,\n2104731913,\n1647629177,\n1691336604,\n922114202,\n170715530,\n1608833393,\n62657989,\n1140989235,\n381784875,\n928003604,\n449509021,\n1057208185,\n1239816707,\n525522922,\n476962140,\n102897870,\n132620570,\n419788154,\n2095057491,\n1240747817,\n1271689397,\n973007445,\n1380110056,\n1021668229,\n12064370,\n1186917580,\n1017163094,\n597085928,\n2018803520,\n1795688603,\n1722115921,\n2015264326,\n506263638,\n1002517905,\n1229603330,\n1376031959,\n763839898,\n1970623926,\n1109937345,\n524780807,\n1976131071,\n905940439,\n1313298413,\n772929676,\n1578848328,\n1108240025,\n577439381,\n1293318580,\n1512203375,\n371003697,\n308046041,\n320070446,\n1252546340,\n568098497,\n1341794814,\n1922466690,\n480833267,\n1060838440,\n969079660,\n1836468543,\n2049091118,\n2023431210,\n383830867,\n2112679659,\n231203270,\n1551220541,\n1377927987,\n275637462,\n2110145570,\n1700335604,\n738389040,\n1688841319,\n1506456297,\n1243730675,\n258043479,\n599084776,\n41093802,\n792486733,\n1897397356,\n28077829,\n1520357900,\n361516586,\n1119263216,\n209458355,\n45979201,\n363681532,\n477245280,\n2107748241,\n601938891,\n244572459,\n1689418013,\n1141711990,\n1485744349,\n1181066840,\n1950794776,\n410494836,\n1445347454,\n2137242950,\n852679640,\n1014566730,\n1999335993,\n1871390758,\n1736439305,\n231222289,\n603972436,\n783045542,\n370384393,\n184356284,\n709706295,\n1453549767,\n591603172,\n768512391,\n854125182,\n),\ndescription=\"Random gear vector\",\n)\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformanc_critical","title":"conformanc_critical  <code>module-attribute</code>","text":"<pre><code>conformanc_critical = {\n\"meta_trim_name\",\n\"meta_trim_description\",\n\"meta_ngram_size_text\",\n\"meta_ngram_size_bytes\",\n\"text_ngram_size\",\n\"text_unicode_filter\",\n\"text_newlines\",\n\"data_avg_chunk_size\",\n\"cdc_gear\",\n}\n</code></pre>"},{"location":"options/options/#iscc_core.options.has_logged_confromance","title":"has_logged_confromance  <code>module-attribute</code>","text":"<pre><code>has_logged_confromance = False\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformance_check_options","title":"conformance_check_options","text":"<pre><code>conformance_check_options(opts)\n</code></pre> <p>Check and log if options have non-default conformance critical values</p>"},{"location":"options/options/#iscc_core.options.core_opts","title":"core_opts  <code>module-attribute</code>","text":"<pre><code>core_opts = CoreOptions()\n</code></pre>"},{"location":"options/options/#iscc_core.options.conformant_options","title":"conformant_options  <code>module-attribute</code>","text":"<pre><code>conformant_options = conformance_check_options(core_opts)\n</code></pre>"},{"location":"units/","title":"ISCC - UNITs","text":"<p>A standard ISCC-CODE is build from multiple ISCC-UNITs. Each unit serve a different purpose.</p>"},{"location":"units/code_data/","title":"ISCC - Data-Code","text":"<p>A similarity perserving hash for binary data (soft hash).</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code","title":"<code>gen_data_code(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create a similarity preserving ISCC Data-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.gen_data_code_v0","title":"<code>gen_data_code_v0(stream, bits = ic.core_opts.data_bits)</code>","text":"<p>Create an ISCC Data-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Data-Code (default 64).</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.soft_hash_data_v0","title":"<code>soft_hash_data_v0(stream)</code>","text":"<p>Create a similarity preserving Data-Hash digest</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Input data stream.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Data-Hash (soft-hash) digest used as body for Data-Code</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0","title":"<code>DataHasherV0</code>","text":"<p>Incremental Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.__init__","title":"<code>__init__(data = None)</code>","text":"<p>Create a DataHasher</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Optional[Data]</code> <p>initial payload for hashing.</p> <code>None</code>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Data-Hash generator.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.digest","title":"<code>digest()</code>","text":"<p>Calculate 256-bit minhash digest from feature hashes.</p>"},{"location":"units/code_data/#iscc_core.code_data.DataHasherV0.code","title":"<code>code(bits = ic.core_opts.data_bits)</code>","text":"<p>Encode digest as an ISCC Data-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Data-Code</p> <code>ic.core_opts.data_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Data-Code</p>"},{"location":"units/code_flake/","title":"ISCC - Flake-Code","text":"<p>A unique, time-sorted identifier composed of an 48-bit timestamp and 16 to 208 bit randomness.</p> <p>The ISCC Flake-Code is a unique identifier for distributed ID generation. The 64-bit version can be used as efficient surrogate key in database systems. It has guaranteed uniqueness if generated from a singele process and is time sortable in integer and base32hex representation. The 128-bit version is a K-sortable, globally unique identifier for use in distributed systems and is compatible with UUID.</p> <p>Example</p> <pre><code>&gt;&gt;&gt; import iscc_core as ic\n&gt;&gt;&gt; ic.gen_flake_code(bits=64)\n{'iscc': 'ISCC:OAAQC7YN7PG2XOR4'}\n&gt;&gt;&gt; ic.gen_flake_code(bits=128)\n{'iscc': 'ISCC:OABQC7YN7RJGUUTLKDSKBXO25MA5E'}\n# Or use the convenience Flake class for easy access to different representations\n&gt;&gt;&gt; flake = ic.Flake(bits=64)\n&gt;&gt;&gt; flake.iscc\n'ISCC:OAAQC7YOADBZYNF7'\n&gt;&gt;&gt; flake.time\n'2022-02-18T18:03:25.468'\n&gt;&gt;&gt; flake.int\n107820312524764351\n&gt;&gt;&gt; flake.string\n'05VGS063JGQBU'\n</code></pre>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code","title":"<code>gen_flake_code(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest standard algorithm</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.gen_flake_code_v0","title":"<code>gen_flake_code_v0(bits = ic.core_opts.flake_bits)</code>","text":"<p>Create an ISCC Flake-Code with the latest algorithm v0</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Target bit-length of generated Flake-Code</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Flake-Code</p>"},{"location":"units/code_flake/#iscc_core.code_flake.uid_flake_v0","title":"<code>uid_flake_v0(ts = None, bits = ic.core_opts.flake_bits)</code>","text":"<p>Generate time and randomness based Flake-Hash</p> <p>Parameters:</p> Name Type Description Default <code>ts</code> <code>Optional[float]</code> <p>Unix timestamp (defaults to current time)</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length resulting Flake-Code (multiple of 32)</p> <code>ic.core_opts.flake_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Flake-Hash digest</p>"},{"location":"units/code_instance/","title":"ISCC - Instance-Code","text":"<p>A data checksum.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code","title":"<code>gen_instance_code(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with properties: iscc, datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.gen_instance_code_v0","title":"<code>gen_instance_code_v0(stream, bits = ic.core_opts.instance_bits)</code>","text":"<p>Create an ISCC Instance-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for Instance-Code generation</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Instance-Code and properties: datahash, filesize</p>"},{"location":"units/code_instance/#iscc_core.code_instance.hash_instance_v0","title":"<code>hash_instance_v0(stream)</code>","text":"<p>Create 256-bit hash digest for the Instance-Code body</p> <p>Parameters:</p> Name Type Description Default <code>stream</code> <code>Stream</code> <p>Binary data stream for hash generation.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit Instance-Hash digest used as body of Instance-Code</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0","title":"<code>InstanceHasherV0</code>","text":"<p>Incremental Instance-Hash generator.</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.push","title":"<code>push(data)</code>","text":"<p>Push data to the Instance-Hash generator.</p> <p>Parameters:</p> Name Type Description Default <code>data</code> <code>Data</code> <p>Data to be hashed</p> required"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.digest","title":"<code>digest()</code>","text":"<p>Return Instance-Hash</p> <p>Returns:</p> Type Description <code>bytes</code> <p>Instance-Hash digest</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.multihash","title":"<code>multihash()</code>","text":"<p>Return blake3 multihash</p> <p>Returns:</p> Type Description <code>str</code> <p>Blake3 hash as 256-bit multihash</p>"},{"location":"units/code_instance/#iscc_core.code_instance.InstanceHasherV0.code","title":"<code>code(bits = ic.core_opts.instance_bits)</code>","text":"<p>Encode digest as an ISCC Instance-Code unit.</p> <p>Parameters:</p> Name Type Description Default <code>bits</code> <code>int</code> <p>Number of bits for the ISCC Instance-Code</p> <code>ic.core_opts.instance_bits</code> <p>Returns:</p> Type Description <code>str</code> <p>ISCC Instance-Code</p>"},{"location":"units/code_meta/","title":"ISCC - Meta-Code","text":"<p>A similarity preserving hash for digital asset metadata.</p>"},{"location":"units/code_meta/#purpose","title":"Purpose","text":"<p>The Meta-Code is the first possible (optional) unit of an ISCC-CODE. It is calculated from the metadata of a digital asset. The primary purpose of the Meta-Code is to aid the discovery of digital assets with similar metadata and the detection of metadata anomalies. As a secondary function, Meta-Code processing also creates a secure Meta-Hash for cryptogrpahic binding purposes.</p>"},{"location":"units/code_meta/#inputs","title":"Inputs","text":"<p>The metadata supplied for Meta-Code calculation is called Seed-Metadata. Seed-Metadata has 3 possible elements:</p> <ul> <li>name (required): The name or title of the work manifested by the digital asset.</li> <li>description (optional): A disambiguating textual description of the digital asset.</li> <li>meta (optional): Industry-sector or use-case specific metadata, encoded as     Data-URL (RFC-2397).</li> </ul> <p>Note</p> <p>Due to the broad applicability of the ISCC we do not prescribe a particular schema for the <code>meta</code>-element. Instead we use the Data-URL format because it can encode and self-describe any conceivable metadata in a sufficently machine-interpretable form at any desired specificity.</p> <p>Data-URL Examples:</p> <ul> <li>Metadata is \"some\" JSON: <code>data:application/json;base64,&lt;data&gt;</code></li> <li>Metadata is JSON-LD: <code>data:application/ld+json;base64,&lt;data&gt;</code></li> <li>Metadata is \"some\" XML: <code>data:application/xml;base64,&lt;data&gt;</code></li> <li>Metadata is MARC21 XML: <code>data:application/marcxml+xml;base64,&lt;data&gt;</code></li> <li>Metadata is IPTC NewsML: <code>data:application/vnd.iptc.g2.newsitem+xml;base64,&lt;data&gt;</code></li> </ul> <p>Data-URLs are also supported by all major internet browsers.</p>"},{"location":"units/code_meta/#processing","title":"Processing","text":""},{"location":"units/code_meta/#meta-code","title":"Meta-Code","text":"<p>The first 32-bits of a Meta-Code are calculated as a simliarity hash from the <code>name</code> field. The second 32-bits are also calculated from the <code>name</code> field if no other input was supplied. If <code>description</code> is suplied but no <code>meta</code>, the <code>description</code> will be used for the second 32-bits. If <code>meta</code> is supplied it will be used in favour of <code>description</code> for the second 32-bits.</p> <pre><code>flowchart LR\n    B{name?}\n    B --&gt;|Yes| J[\"P1 = SH(name)\"]\n    J --&gt; D{meta?}\n    D --&gt;|Yes| F[\"Meta-Code = P1 + SH(meta)\"]\n    D --&gt;|No| G{description?}\n    G --&gt;|Yes| H[\"Meta-Code = P1 + SH(description)\"]\n    G --&gt;|No| I[\"Meta-Code = P1\"]\n    B --&gt;|No| E[Skip Meta-Code]</code></pre> <p>Note</p> <p>To support automation and reproducibility, applications that generate ISCCs, should prioritize metadata that is automatically extracted from the digital asset.</p> <p>If embedded metadata is not available or known to be unreliable an application should rely on external metadata or explicitly ask users to supply at least the <code>name</code>-field. Applications should then first embed metadata into the asset before calculating the ISCC-CODE. This ensures that the embedded metadata is bound to the asset by the Instance-Code.</p> <p>If neither embedded nor external metadata is available, the application may resort to use the filename of the digital asset as value for the <code>name</code>-field. If no value can be determined for the <code>name</code>-field, an application shall skip generation of a Meta-Code and create an ISCC-CODE without a Meta-Code.</p>"},{"location":"units/code_meta/#meta-hash","title":"Meta-Hash","text":"<p>In addition to the Meta-Code we also create a cryptographic hash (the Meta-Hash) of the supplied Seed-Metadata. It is used to securely bind metadata to the digital asset.</p> <pre><code>flowchart LR\n    N{name?}\n    N --&gt;|Yes| A{meta?}\n    N --&gt;|No| E[Skip Meta-Hash]\n    A --&gt;|Yes| H[\"Meta-Hash = H(meta)\"]\n    A --&gt;|No| B{description?}\n    B --&gt;|Yes| HND[\"Meta-Hash = H(name + description)\"]\n    B --&gt;|No| HN[\"Meta-Hash = H(name)\"]</code></pre>"},{"location":"units/code_meta/#functions","title":"Functions","text":""},{"location":"units/code_meta/#iscc_core.code_meta.gen_meta_code_v0","title":"<code>gen_meta_code_v0(name, description = None, meta = None, bits = ic.core_opts.meta_bits)</code>","text":"<p>Create an ISCC Meta-Code with the algorithm version 0.</p> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Name or title of the work manifested by the digital asset</p> required <code>description</code> <code>Optional[str]</code> <p>Optional description for disambiguation</p> <code>None</code> <code>meta</code> <code>Optional[Union[dict,str]</code> <p>Dict or Data-URL string with extended metadata</p> <code>None</code> <code>bits</code> <code>int</code> <p>Bit-length of resulting Meta-Code (multiple of 64)</p> <code>ic.core_opts.meta_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with possible fields: iscc, name, description, metadata, metahash</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def gen_meta_code_v0(name, description=None, meta=None, bits=ic.core_opts.meta_bits):\n# type: (str, Optional[str], Optional[ic.Meta], int) -&gt; dict\n\"\"\"\n    Create an ISCC Meta-Code with the algorithm version 0.\n    :param str name: Name or title of the work manifested by the digital asset\n    :param Optional[str] description: Optional description for disambiguation\n    :param Optional[Union[dict,str] meta: Dict or Data-URL string with extended metadata\n    :param int bits: Bit-length of resulting Meta-Code (multiple of 64)\n    :return: ISCC object with possible fields: iscc, name, description, metadata, metahash\n    :rtype: dict\n    \"\"\"\n# 1. Normalize `name`\nname = \"\" if name is None else name\nname = text_clean(name)\nname = text_remove_newlines(name)\nname = text_trim(name, ic.core_opts.meta_trim_name)\nif not name:\nraise ValueError(\"Meta-Code requires non-empty name element (after normalization)\")\n# 2. Normalize `description`\ndescription = \"\" if description is None else description\ndescription = text_clean(description)\ndescription = text_trim(description, ic.core_opts.meta_trim_description)\n# Calculate meta_code, metahash, and output metadata values for the different input cases\nif meta:\nif isinstance(meta, str):\n# Data-URL expected\ndurl = meta\npayload = DataURL.from_url(durl).data\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = durl\nelif isinstance(meta, dict):\npayload = jcs.canonicalize(meta)\nmeta_code_digest = soft_hash_meta_v0(name, payload)\nmetahash = ic.multi_hash_blake3(payload)\nmedia_type = \"application/ld+json\" if \"@context\" in meta else \"application/json\"\ndurl_obj = DataURL.from_data(media_type, base64_encode=True, data=payload)\nmetadata_value = durl_obj.url\nelse:\nraise TypeError(f\"metadata must be Data-URL string or dict not {type(meta)}\")\nelse:\npayload = \" \".join((name, description)).strip().encode(\"utf-8\")\nmeta_code_digest = soft_hash_meta_v0(name, description)\nmetahash = ic.multi_hash_blake3(payload)\nmetadata_value = None\nmeta_code = ic.encode_component(\nmtype=ic.MT.META,\nstype=ic.ST.NONE,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=meta_code_digest,\n)\niscc = \"ISCC:\" + meta_code\n# Build result\nresult = {\"iscc\": iscc}\nif name:\nresult[\"name\"] = name\nif description:\nresult[\"description\"] = description\nif metadata_value:\nresult[\"meta\"] = metadata_value\nresult[\"metahash\"] = metahash\nreturn result\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.soft_hash_meta_v0","title":"<code>soft_hash_meta_v0(name, extra = None)</code>","text":"<p>Calculate simmilarity preserving 256-bit hash digest from asset metadata.</p> <p>Textual input should be stripped of markup, normalized and trimmed before hashing. Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be serialized in a canonical form (for example JCS for JSON)</p> <p>Note</p> <p>The processing algorithm depends on the type of the <code>extra</code> input. If the <code>extra</code> field is supplied and non-empty, we create separate hashes for <code>name</code> and <code>extra</code> and interleave them in 32-bit chunks:</p> <ul> <li> <p>If the <code>extra</code> input is <code>None</code> or an empty <code>str</code>/<code>bytes</code> object the Meta-Hash will be generated from the <code>name</code>-field only.</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty text string (str) the string is lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).</p> </li> <li> <p>If the <code>extra</code>-input is a non-empty bytes object the processing is done bytewise.</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>name</code> <code>str</code> <p>Title of the work manifested in the digital asset</p> required <code>extra</code> <code>Union[str,bytes,None]</code> <p>Additional metadata for disambiguation</p> <code>None</code> <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit simhash digest for Meta-Code</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def soft_hash_meta_v0(name, extra=None):\n# type: (str, Union[str,bytes,None]) -&gt; bytes\n\"\"\"\n    Calculate simmilarity preserving 256-bit hash digest from asset metadata.\n    Textual input should be stripped of markup, normalized and trimmed before hashing.\n    Bytes input can be any serialized metadata (JSON, XML, Image...). Metadata should be\n    serialized in a canonical form (for example\n    [JCS](https://tools.ietf.org/id/draft-rundgren-json-canonicalization-scheme-00.html) for JSON)\n    !!! note\n        The processing algorithm depends on the type of the `extra` input.\n        If the `extra` field is supplied and non-empty, we create separate hashes for\n        `name` and `extra` and interleave them in 32-bit chunks:\n        - If the `extra` input is `None` or an empty `str`/`bytes` object the Meta-Hash will\n        be generated from the `name`-field only.\n        - If the `extra`-input is a non-empty **text** string (str) the string is\n        lower-cased and the processing units are utf-8 endoded characters (possibly multibyte).\n        - If the `extra`-input is a non-empty **bytes** object the processing is done bytewise.\n    :param str name: Title of the work manifested in the digital asset\n    :param Union[str,bytes,None] extra: Additional metadata for disambiguation\n    :return: 256-bit simhash digest for Meta-Code\n    :rtype: bytes\n    \"\"\"\nname = ic.text_collapse(name)\nname_n_grams = ic.sliding_window(name, width=ic.core_opts.meta_ngram_size_text)\nname_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in name_n_grams]\nsimhash_digest = ic.alg_simhash(name_hash_digests)\nif extra in {None, \"\", b\"\"}:\nreturn simhash_digest\nelse:\n# Augment with interleaved hash for extra metadata\nif isinstance(extra, bytes):\n# Raw bytes are handled per byte\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_bytes)\nextra_hash_digests = [blake3(ngram).digest() for ngram in extra_n_grams]\nelif isinstance(extra, str):\n# Text is collapsed and handled per character (multibyte)\nextra = ic.text_collapse(extra)\nextra_n_grams = ic.sliding_window(extra, width=ic.core_opts.meta_ngram_size_text)\nextra_hash_digests = [blake3(s.encode(\"utf-8\")).digest() for s in extra_n_grams]\nelse:\nraise ValueError(\"parameter `extra` must be of type str or bytes!\")\nextra_simhash_digest = ic.alg_simhash(extra_hash_digests)\n# Interleave first half of name and extra simhashes in 32-bit chunks\nchunks_simhash_digest = sliced(simhash_digest[:16], 4)\nchunks_extra_simhash_digest = sliced(extra_simhash_digest[:16], 4)\ninterleaved = interleave(chunks_simhash_digest, chunks_extra_simhash_digest)\nsimhash_digest = bytearray()\nfor chunk in interleaved:\nsimhash_digest += chunk\nsimhash_digest = bytes(simhash_digest)\nreturn simhash_digest\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_clean","title":"<code>text_clean(text)</code>","text":"<p>Clean text for display.</p> <ul> <li>Normalize with NFKC normalization.</li> <li>Remove Control Characters (except newlines)</li> <li>Reduce multiple consecutive newlines to a maximum of two newlines</li> <li>Strip leading and trailing whitespace</li> </ul> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_clean(text):\n# type: (str) -&gt; str\n\"\"\"\n    Clean text for display.\n    - Normalize with NFKC normalization.\n    - Remove Control Characters (except newlines)\n    - Reduce multiple consecutive newlines to a maximum of two newlines\n    - Strip leading and trailing whitespace\n    \"\"\"\n# Unicode normalize\ntext = unicodedata.normalize(\"NFKC\", text)\n# Remove control characters\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] != \"C\" or ch in ic.core_opts.text_newlines\n)\n# Collapse more than two consecutive newlines\nchars = []\nnewline_count = 0\nfor c in text:\nif c in ic.core_opts.text_newlines:\nif newline_count &lt; 2:\nchars.append(\"\\n\")\nnewline_count += 1\ncontinue\nelse:\nnewline_count = 0\nchars.append(c)\ntext = \"\".join(chars)\nreturn text.strip()\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_remove_newlines","title":"<code>text_remove_newlines(text)</code>","text":"<p>Remove newlines.</p> <p>The <code>name</code> field serves as a displayable title. We remove newlines and leading and trailing whitespace. We also collapse consecutive spaces to single spaces.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <p>Text for newline removal</p> required <p>Returns:</p> Type Description <code>str</code> <p>Single line of text</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_remove_newlines(text):\n# type: (str) -&gt; str\n\"\"\"\n    Remove newlines.\n    The `name` field serves as a displayable title. We remove newlines and leading and trailing\n    whitespace. We also collapse consecutive spaces to single spaces.\n    :param text: Text for newline removal\n    :return: Single line of text\n    :rtype: str\n    \"\"\"\nreturn \" \".join(text.split())\n</code></pre>"},{"location":"units/code_meta/#iscc_core.code_meta.text_trim","title":"<code>text_trim(text, nbytes)</code>","text":"<p>Trim text such that its utf-8 encoded size does not exceed <code>nbytes</code>.</p> Source code in <code>iscc_core\\code_meta.py</code> <pre><code>def text_trim(text, nbytes):\n# type: (str, int) -&gt; str\n\"\"\"Trim text such that its utf-8 encoded size does not exceed `nbytes`.\"\"\"\nreturn text.encode(\"utf-8\")[:nbytes].decode(\"utf-8\", \"ignore\").strip()\n</code></pre>"},{"location":"units/content/","title":"ISCC - Content-Codes","text":""},{"location":"units/content/code_content_audio/","title":"ISCC - Audio-Code","text":"<p>A similarity preserving hash for audio content (soft hash).</p> <p>Creates an ISCC object that provides an <code>iscc</code>-field with an Audio-Code and a <code>duration</code>-field.</p> <p>The Content-Code Audio is generated from a Chromaprint fingerprint provided as a vector of 32-bit signed integers. The iscc-sdk uses fpcalc to extract Chromaprint vectors with the following command line parameters:</p> <p><code>$ fpcalc -raw -json -signed -length 0 myaudiofile.mp3</code></p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code","title":"<code>gen_audio_code(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.gen_audio_code_v0","title":"<code>gen_audio_code_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create an ISCC Content-Code Audio with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Content-Code Audio (multiple of 64)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Audio</p>"},{"location":"units/content/code_content_audio/#iscc_core.code_content_audio.soft_hash_audio_v0","title":"<code>soft_hash_audio_v0(cv, bits = ic.core_opts.audio_bits)</code>","text":"<p>Create audio similarity hash from a chromaprint vector.</p> <p>Parameters:</p> Name Type Description Default <code>cv</code> <code>Iterable[int]</code> <p>Chromaprint vector</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting similarity hash (multiple of 32)</p> <code>ic.core_opts.audio_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Audio-Hash digest</p>"},{"location":"units/content/code_content_image/","title":"ISCC - Image-Code","text":"<p>A similarity preserving perceptual hash for images.</p> <p>The ISCC Content-Code Image is created by calculating a discrete cosine transform on normalized image-pixels and comparing the values from the upper left area of the dct-matrix against their median values to set the hash-bits.</p> <p>Images must be normalized before using gen_image_code. Prepare images as follows:</p> <ul> <li>Transpose image according to EXIF Orientation</li> <li>Add white background to image if it has alpha transparency</li> <li>Crop empty borders of image</li> <li>Convert image to grayscale</li> <li>Resize image to 32x32</li> <li>Flatten 32x32 matrix to an array of 1024 grayscale (uint8) pixel values</li> </ul>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code","title":"<code>gen_image_code(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values).</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.gen_image_code_v0","title":"<code>gen_image_code_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Create an ISCC Content-Code Image with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> <p>Normalized image pixels (32x32 flattened gray values)</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Content-Code Image (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>ISCC</code> <p>ISCC object with Content-Code Image.</p>"},{"location":"units/content/code_content_image/#iscc_core.code_content_image.soft_hash_image_v0","title":"<code>soft_hash_image_v0(pixels, bits = ic.core_opts.image_bits)</code>","text":"<p>Calculate image hash from normalized grayscale pixel sequence of length 1024.</p> <p>Parameters:</p> Name Type Description Default <code>pixels</code> <code>Sequence[int]</code> required <code>bits</code> <code>int</code> <p>Bit-length of image hash (default 64).</p> <code>ic.core_opts.image_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving Image-Hash digest.</p>"},{"location":"units/content/code_content_mixed/","title":"ISCC - Mixed Code","text":"<p>A similarity hash for mixed media content.</p> <p>Creates an ISCC object that provides a <code>iscc</code>-field a Mixed-Code and a <code>parts</code>-field that lists the input codes.</p> <p>Many digital assets embed multiple assets of different mediatypes in a single file. Text documents may include images, video includes audio in most cases. The ISCC Content-Code-Mixed encodes the similarity of a collection of assets of the same or different mediatypes that may occur in a multimedia asset.</p> <p>Applications that create mixed Content-Codes must be capable to extract embedded assets and create individual Content-Codes per asset.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code","title":"<code>gen_mixed_code(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code Mixed with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.gen_mixed_code_v0","title":"<code>gen_mixed_code_v0(codes, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create an ISCC Content-Code-Mixed with algorithm v0.</p> <p>If the provided codes are of mixed length they are stripped to <code>bits</code> length for calculation.</p> <p>Parameters:</p> Name Type Description Default <code>codes</code> <code>Iterable[str]</code> <p>a list of Content-Codes.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Content-Code Mixed.</p>"},{"location":"units/content/code_content_mixed/#iscc_core.code_content_mixed.soft_hash_codes_v0","title":"<code>soft_hash_codes_v0(cc_digests, bits = ic.core_opts.mixed_bits)</code>","text":"<p>Create a similarity hash from multiple Content-Code digests.</p> <p>The similarity hash is created from the bodies of the input codes with the first byte of the code-header prepended.</p> <p>All codes must be of main-type CONTENT and have a minimum length of <code>bits</code>.</p> <p>Parameters:</p> Name Type Description Default <code>cc_digests</code> <code>Sequence[bytes]</code> <p>a list of Content-Code digests.</p> required <code>bits</code> <code>int</code> <p>Target bit-length of generated Content-Code-Mixed.</p> <code>ic.core_opts.mixed_bits</code> <p>Returns:</p> Type Description <code>bytes</code> <p>Similarity preserving byte hash.</p>"},{"location":"units/content/code_content_text/","title":"ISCC - Text Code","text":"<p>A similarity preserving hash for plain-text content (soft hash).</p> <p>The ISCC Text-Code is generated from plain-text that has been extracted from a media assets.</p> <p>Warning</p> <p>Plain-text extraction from documents in various formats (especially PDF) may yield very diffent results depending on the extraction tools being used. The iscc-sdk uses Apache Tika to extract text from documents for Text-Code generation.</p> <p>Algorithm overview</p> <ul> <li>Apply <code>text_collapse</code> function to text input</li> <li>Count characters of collapsed text</li> <li>Apply <code>soft_hash_text_v0</code> to collapsed text</li> </ul>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.gen_text_code_v0","title":"<code>gen_text_code_v0(text, bits = ic.core_opts.text_bits)</code>","text":"<p>Create an ISCC Text-Code with algorithm v0.</p> <p>Note</p> <p>Any markup (like HTML tags or markdown) should be removed from the plain-text before passing it to this function.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Text for Text-Code creation</p> required <code>bits</code> <code>int</code> <p>Bit-length of ISCC Code Hash (default 64)</p> <code>ic.core_opts.text_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC schema instance with Text-Code and an aditional property <code>characters</code></p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def gen_text_code_v0(text, bits=ic.core_opts.text_bits):\n# type: (str, int) -&gt; dict\n\"\"\"\n    Create an ISCC Text-Code with algorithm v0.\n    !!! note\n        Any markup (like HTML tags or markdown) should be removed from the plain-text\n        before passing it to this function.\n    :param str text: Text for Text-Code creation\n    :param int bits: Bit-length of ISCC Code Hash (default 64)\n    :return: ISCC schema instance with Text-Code and an aditional property `characters`\n    :rtype: dict\n    \"\"\"\ntext = text_collapse(text)\ncharacters = len(text)\ndigest = soft_hash_text_v0(text)\ntext_code = ic.encode_component(\nmtype=ic.MT.CONTENT,\nstype=ic.ST_CC.TEXT,\nversion=ic.VS.V0,\nbit_length=bits,\ndigest=digest,\n)\niscc = \"ISCC:\" + text_code\nreturn dict(iscc=iscc, characters=characters)\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.text_collapse","title":"<code>text_collapse(text)</code>","text":"<p>Normalize and simplify text for similarity hashing.</p> <ul> <li>Decompose with NFD normalization.</li> <li>Remove all whitespace characters and convert text to lower case</li> <li>Filter control characters, marks (diacritics), and punctuation</li> <li>Recombine with NFKC normalization.</li> </ul> <p>Note</p> <p>See: Unicode normalization.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be collapsed.</p> required <p>Returns:</p> Type Description <code>str</code> <p>Collapsed plain text.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def text_collapse(text):\n# type: (str) -&gt; str\n\"\"\"\n    Normalize and simplify text for similarity hashing.\n    - Decompose with NFD normalization.\n    - Remove all whitespace characters and convert text to lower case\n    - Filter control characters, marks (diacritics), and punctuation\n    - Recombine with NFKC normalization.\n    !!! note\n        See: [Unicode normalization](https://unicode.org/reports/tr15/).\n    :param str text: Plain text to be collapsed.\n    :return: Collapsed plain text.\n    :rtype: str\n    \"\"\"\n# Decompose with NFD\ntext = unicodedata.normalize(\"NFD\", text)\n# Remove all whitespace and convert text to lower case\ntext = \"\".join(text.split()).lower()\n# Filter control characters, marks (diacritics), and punctuation\ntext = \"\".join(\nch for ch in text if unicodedata.category(ch)[0] not in ic.core_opts.text_unicode_filter\n)\n# Recombine\ntext = unicodedata.normalize(\"NFKC\", text)\nreturn text\n</code></pre>"},{"location":"units/content/code_content_text/#iscc_core.code_content_text.soft_hash_text_v0","title":"<code>soft_hash_text_v0(text)</code>","text":"<p>Creates a 256-bit similarity preserving hash for text input with algorithm v0.</p> <ul> <li>Slide over text with a   <code>text_ngram_size</code> wide window   and create <code>xxh32</code> hashes</li> <li>Create a <code>minhash_256</code> from the hashes generated   in the previous step.</li> </ul> <p>Note</p> <p>Before passing text to this function it must be:</p> <ul> <li>stripped of markup</li> <li>normalized</li> <li>stripped of whitespace</li> <li>lowercased</li> </ul> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>Plain text to be hashed.</p> required <p>Returns:</p> Type Description <code>bytes</code> <p>256-bit similarity preserving byte hash.</p> Source code in <code>iscc_core\\code_content_text.py</code> <pre><code>def soft_hash_text_v0(text):\n# type: (str) -&gt; bytes\n\"\"\"\n    Creates a 256-bit similarity preserving hash for text input with algorithm v0.\n    - Slide over text with a\n      [`text_ngram_size`][iscc_core.options.CoreOptions.text_ngram_size] wide window\n      and create [`xxh32`](https://cyan4973.github.io/xxHash/) hashes\n    - Create a [`minhash_256`][iscc_core.minhash.alg_minhash_256] from the hashes generated\n      in the previous step.\n    !!! note\n        Before passing text to this function it must be:\n        - stripped of markup\n        - normalized\n        - stripped of whitespace\n        - lowercased\n    :param str text: Plain text to be hashed.\n    :return: 256-bit similarity preserving byte hash.\n    :rtype: bytes\n    \"\"\"\nngrams = ic.sliding_window(text, ic.core_opts.text_ngram_size)\nfeatures = [xxhash.xxh32_intdigest(s.encode(\"utf-8\")) for s in ngrams]\nhash_digest = ic.alg_minhash_256(features)\nreturn hash_digest\n</code></pre>"},{"location":"units/content/code_content_video/","title":"ISCC - Video-Code","text":"<p>A similarity preserving hash for video content</p> <p>The Content-Code Video is generated from MPEG-7 video frame signatures. The iscc-sdk uses ffmpeg to extract frame signatures with the following command line parameters:</p> <p><code>$ ffmpeg -i video.mpg -vf fps=fps=5,signature=format=xml:filename=sig.xml -f null -</code></p> <p>The relevant frame signatures can be parsed from the following elements in sig.xml:</p> <p><code>&lt;FrameSignature&gt;0  0  0  1  0  0  1  0  1  1  0  0  1  1 ...&lt;/FrameSignature&gt;</code></p> <p>Tip</p> <p>It is also possible to extract the signatures in a more compact binary format. But the format requires a custom binary parser to decode the frame signaturs.</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code","title":"<code>gen_video_code(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with the latest standard algorithm.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Instance-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.gen_video_code_v0","title":"<code>gen_video_code_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Create an ISCC Video-Code with algorithm v0.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>Sequence of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code> <p>Returns:</p> Type Description <code>dict</code> <p>ISCC object with Video-Code</p>"},{"location":"units/content/code_content_video/#iscc_core.code_content_video.soft_hash_video_v0","title":"<code>soft_hash_video_v0(frame_sigs, bits = ic.core_opts.video_bits)</code>","text":"<p>Compute video hash v0 from MP7 frame signatures.</p> <p>Parameters:</p> Name Type Description Default <code>frame_sigs</code> <code>ic.FrameSig</code> <p>2D matrix of MP7 frame signatures</p> required <code>bits</code> <code>int</code> <p>Bit-length of resulting Video-Code (multiple of 64)</p> <code>ic.core_opts.video_bits</code>"},{"location":"utilities/utils/","title":"ISCC - Utilities","text":""},{"location":"utilities/utils/#iscc_core.utils.json_canonical","title":"<code>json_canonical(obj)</code>","text":"<p>Canonical, deterministic serialization of ISCC metadata.</p> <p>We serialize ISCC metadata in a deterministic/reproducible manner by using JCS (RFC 8785) canonicalization.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def json_canonical(obj):\n# type: (Any) -&gt; bytes\n\"\"\"\n    Canonical, deterministic serialization of ISCC metadata.\n    We serialize ISCC metadata in a deterministic/reproducible manner by using\n    [JCS (RFC 8785)](https://datatracker.ietf.org/doc/html/rfc8785) canonicalization.\n    \"\"\"\nser = jcs.canonicalize(obj)\ndes = json.loads(ser)\nif des != obj:\nraise ValueError(f\"Not canonicalizable {obj} round-trips to {des}\")\nreturn ser\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.sliding_window","title":"<code>sliding_window(seq, width)</code>","text":"<p>Generate a sequence of equal \"width\" slices each advancing by one elemnt.</p> <p>All types that have a length and can be sliced are supported (list, tuple, str ...). The result type matches the type of the input sequence. Fragment slices smaller than the width at the end of the sequence are not produced. If \"witdh\" is smaller than the input sequence than one element will be returned that is shorter than the requested width.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>Sequence</code> <p>Sequence of values to slide over</p> required <code>width</code> <code>int</code> <p>Width of sliding window in number of items</p> required <p>Returns:</p> Type Description <code>Generator</code> <p>A generator of window sized items</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def sliding_window(seq, width):\n# type: (Sequence, int) -&gt; Generator\n\"\"\"\n    Generate a sequence of equal \"width\" slices each advancing by one elemnt.\n    All types that have a length and can be sliced are supported (list, tuple, str ...).\n    The result type matches the type of the input sequence.\n    Fragment slices smaller than the width at the end of the sequence are not produced.\n    If \"witdh\" is smaller than the input sequence than one element will be returned that\n    is shorter than the requested width.\n    :param Sequence seq: Sequence of values to slide over\n    :param int width: Width of sliding window in number of items\n    :returns: A generator of window sized items\n    :rtype: Generator\n    \"\"\"\nif width &lt; 2:\nraise AssertionError(\"Sliding window width must be 2 or bigger.\")\nidx = range(max(len(seq) - width + 1, 1))\nreturn (seq[i : i + width] for i in idx)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_compare","title":"<code>iscc_compare(a, b)</code>","text":"<p>Calculate separate hamming distances of compatible components of two ISCCs</p> <p>Returns:</p> Type Description <code>dict</code> <p>A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_compare(a, b):\n# type: (str, str) -&gt; dict\n\"\"\"\n    Calculate separate hamming distances of compatible components of two ISCCs\n    :return: A dict with keys meta_dist, semantic_dist, content_dist, data_dist, instance_match\n    :rtype: dict\n    \"\"\"\nac = [ic.Code(unit) for unit in ic.iscc_decompose(a)]\nbc = [ic.Code(unit) for unit in ic.iscc_decompose(b)]\nresult = {}\nfor ca in ac:\nfor cb in bc:\ncat = (ca.maintype, ca.subtype, ca.version)\ncbt = (cb.maintype, cb.subtype, ca.version)\nif cat == cbt:\nif ca.maintype != ic.MT.INSTANCE:\nresult[ca.maintype.name.lower() + \"_dist\"] = iscc_distance_bytes(\nca.hash_bytes, cb.hash_bytes\n)\nelse:\nresult[\"instance_match\"] = ca.hash_bytes == cb.hash_bytes\nreturn result\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_similarity","title":"<code>iscc_similarity(a, b)</code>","text":"<p>Calculate similarity of ISCC codes as a percentage value (0-100).</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Similarity of ISCC a and b in percent (based on hamming distance)</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_similarity(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate similarity of ISCC codes as a percentage value (0-100).\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Similarity of ISCC a and b in percent (based on hamming distance)\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nhdist = iscc_distance_bytes(a, b)\nnbits = len(a) * 8\nsim = int(((nbits - hdist) / nbits) * 100)\nreturn sim\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance","title":"<code>iscc_distance(a, b)</code>","text":"<p>Calculate hamming distance of ISCC codes.</p> <p>MainType, SubType, Version and Length of the codes must be the same.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distanced in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance(a, b):\n# type: (str, str) -&gt; int\n\"\"\"\n    Calculate hamming distance of ISCC codes.\n    MainType, SubType, Version and Length of the codes must be the same.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Hamming distanced in number of bits.\n    :rtype: int\n    \"\"\"\na, b = iscc_pair_unpack(a, b)\nreturn iscc_distance_bytes(a, b)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_distance_bytes","title":"<code>iscc_distance_bytes(a, b)</code>","text":"<p>Calculate hamming distance for binary hash digests of equal length.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <code>bytes</code> <p>binary hash digest</p> required <code>b</code> <code>bytes</code> <p>binary hash digest</p> required <p>Returns:</p> Type Description <code>int</code> <p>Hamming distance in number of bits.</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_distance_bytes(a, b):\n# type: (bytes, bytes) -&gt; int\n\"\"\"\n    Calculate hamming distance for binary hash digests of equal length.\n    :param bytes a: binary hash digest\n    :param bytes b: binary hash digest\n    :return: Hamming distance in number of bits.\n    :rtype: int\n    \"\"\"\nif len(a) != len(b):\nraise AssertionError(f\"Hash diggest of unequal length: {len(a)} vs {len(b)}\")\nba, bb = bitarray(), bitarray()\nba.frombytes(a)\nbb.frombytes(b)\nreturn count_xor(ba, bb)\n</code></pre>"},{"location":"utilities/utils/#iscc_core.utils.iscc_pair_unpack","title":"<code>iscc_pair_unpack(a, b)</code>","text":"<p>Unpack two ISCC codes and return their body hash digests if their headers match.</p> <p>Headers match if their MainType, SubType, and Version are identical.</p> <p>Parameters:</p> Name Type Description Default <code>a</code> <p>ISCC a</p> required <code>b</code> <p>ISCC b</p> required <p>Returns:</p> Type Description <code>Tuple[bytes, bytes]</code> <p>Tuple with hash digests of a and b</p> <p>Raises:</p> Type Description <code>ValueError</code> <p>If ISCC headers don\u00b4t match</p> Source code in <code>iscc_core\\utils.py</code> <pre><code>def iscc_pair_unpack(a, b):\n# type: (str, str) -&gt; Tuple[bytes, bytes]\n\"\"\"\n    Unpack two ISCC codes and return their body hash digests if their headers match.\n    Headers match if their MainType, SubType, and Version are identical.\n    :param a: ISCC a\n    :param b: ISCC b\n    :return: Tuple with hash digests of a and b\n    :rtype: Tuple[bytes, bytes]\n    :raise ValueError: If ISCC headers don\u00b4t match\n    \"\"\"\na, b = ic.iscc_clean(ic.iscc_normalize(a)), ic.iscc_clean(ic.iscc_normalize(b))\na, b = ic.decode_base32(a), ic.decode_base32(b)\na, b = ic.decode_header(a), ic.decode_header(b)\nif not a[:-1] == b[:-1]:\nraise ValueError(f\"ISCC headers don\u00b4t match: {a}, {b}\")\nreturn a[-1], b[-1]\n</code></pre>"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 2743f7a..0332f99 100755
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,137 +2,137 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://core.iscc.codes/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/changelog/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/conformance/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/constants/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/iscc_code/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/iscc_id/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/iso-reference/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/cdc/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/dct/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/minhash/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/simhash/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/algorithms/wtahash/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/codec/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/includes/abbr/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/options/options/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_data/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_flake/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_instance/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/code_meta/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_audio/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_image/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_mixed/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_text/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/units/content/code_content_video/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://core.iscc.codes/utilities/utils/</loc>
-         <lastmod>2024-01-30</lastmod>
+         <lastmod>2024-03-17</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 510eb56cc21c742251296e66cf6ec326b3c21af8..06fb53da27ea143f47050dc0e67fe7d3aa40c42f 100755
GIT binary patch
literal 379
zcmV->0fhb^iwFo4UH4@I|8r?{Wo=<_E_iKh0M*vLZi6ro0N}k(f!M($t<)b8vh@kt
zIdbd+PW4Y}A0d7F8iLxaU8;itpUvs`WE<gnf3YSzLx||zF5i?@o`F+dqw9A0``c^r
zl<%9ndI%;0WhvRkUEUL&UW6FOv5ZJy<tdgvbRw!gb)r}XMO8lM&0UtRslYXwuiALk
zOjTkkqkMo8qf%vhFGNv`>FI5<jAXrTw$=8bsP2o+V@g@ZZqigQT?cJ|&qhl$U$wXy
zxhljD<1e6V8RI*KH1Ap;taJ)k;vz{$43~5Y4^zmLhMFVH7Xexbz;d@q)A@kZ+qCht
zV(rt4So<uxUdEoaF`{K}97*ytPIrc%2s&kkc@B@qz>nuoluk*%Ej!@gkl3=O_@E_8
z*1*=t4`40C`B0^JBQ0=VXGh?OJ8z!HrE`#jM!zaCTG_#>#1=20S0yI6tZN&eQ3J<q
ZOlXjBiT>daIyVuh`37nIbkcwk000>BxL5!H

literal 378
zcmV-=0fqh_iwFpLySQZn|8r?{Wo=<_E_iKh0M*vLZi6ro0N_1Of!M($?a)?)Y<+@u
zjvV`dQ~i_LM@ZkkhM+cUm+D}^XLC9}*+#fNT&>A25F&cF&$nfjXW*3A=(>IW{`Oit
z<%ecd55YvBEG4_R&wHZtixA^DmJtc8Jjc?9PDItGP87?asLJQO*<|UO3f!Xks*P98
zR3)Y|$_FShDpjWULKL-_p57<RNY?9SSM45)YFj*1DP<YEOH;jc9kc;H8!gd%)#7gC
zrVu}jzkqIKjPDrIylZ{1(kWnxizFQ}T+%5#P9akoYK|~p1ZW`u%iShT=L1r2)5g<^
zwNEQz?X&258GF{oh?c!^B+1h_-5Gu&=#&}eIXs;LKb}8RIwk$K?0|zqV#}K1qn0FD
z16w0MfVB|kLzUu<w7_|t9f2e6ym=m%&QT5;{i?)hWe2MgTfBl^m6+hVu5Ell4IH;I
Yp+UkW`iDR0+(e+}8|BU+Zh#R000IfO@Bjb+