Merge branch 'develop'

Conflicts:
	heatshrink_encoder.c

Merge several PRs and other changes on `develop` over in preparation for
0.4.0 release.

Closes #16.
(Addressed directly by 15ebadd, and indirectly by several other changes.)

.gitignore

@@ -5,3 +5,4 @@ test_heatshrink_static
 *.core
 *.dSYM
 *.exe
+benchmark_out/

CONTRIBUTING.md

@@ -0,0 +1,86 @@
+# Contributing to heatshrink
+
+Thanks for taking time to contribute to heatshrink!
+
+Some issues may be tagged with `beginner` in the issue tracker, those
+should be particularly approachable.
+
+Please send pull requests against the `develop` branch. Changes need to
+be carefully checked for reverese compatibility before merging to
+`master`, since heatshrink is running on devices that may not be easily
+recalled and updated.
+
+
+## Documentation
+
+Improvements to the documentation are welcome. So are requests for
+clarification -- if the docs are unclear or misleading, that's a
+potential source of bugs.
+
+
+## Embedded & Portability Constraints
+
+heatshrink primarily targets embedded / real-time / memory-constrained
+systems, so enhancements that significantly increase memory or code
+space (ROM) requirements are probably out of scope.
+
+Changes that improve portability are welcome, and feedback from running
+on different embedded platforms is appreciated.
+
+
+## Versioning & Compatibility
+
+The versioning format is MAJOR.MINOR.PATCH.
+
+Performance improvements or minor bug fixes that do not break
+compatibility with past releases lead to patch version increases. API
+changes that do not break compatibility lead to minor version increases
+and reset the patch version, and changes that do break compatibility
+lead to a major version increase.
+
+Since heatshrink's compression and decompression sides may be used and
+updated independently, any change to the encoder that cannot be
+correctly decoded by earlier releases (or vice versa) is considered a
+breaking change. Changes to the encoder that lead to different output
+that earlier decoder releases handle correctly (such as pattern
+detection improvements) are *not* breaking changes.
+
+Essentially, improvements to the compression process that older releases
+can't decode correctly will need to wait until the next major release.
+
+
+## LZSS Algorithm
+
+heatshrink uses the [Lempel-Ziv-Storer-Szymanski][LZSS] algorithm for
+compression, with a few important implementation details:
+
+1. The compression and uncompression state machines have been designed
+   to run incrementally - processing can work a few bytes at a time,
+   suspending and resuming as additional data / buffer space becomes
+   available.
+
+2. The optional [indexing technique][index] used to speed up compression
+   is unique to heatshrink, as far as I know.
+
+3. In general, implementation trade-offs have favored low memory usage.
+
+[index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/
+[LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski
+
+
+## Testing
+
+The unit tests are based on [greatest][g], with additional
+property-based tests using [theft][t] (which are currently not built by
+default). greatest tests are preferred for specific new functionality
+and for regression tests, while theft tests are preferred for
+integration tests (e.g. "for any input, compressing and uncompressing it
+should match the original"). Bugs found by theft make for great regression
+tests.
+
+Contributors are encouraged to add tests for any new functionality, and
+in particular to add regression tests for any bugs found.
+
+[g]: https://github.com/silentbicycle/greatest
+[t]: https://github.com/silentbicycle/theft
+

Makefile

@@ -4,12 +4,15 @@ WARN = -Wall -Wextra -pedantic #-Werror
 WARN += -Wmissing-prototypes
 WARN += -Wstrict-prototypes
 WARN += -Wmissing-declarations
-CFLAGS += -std=c99 -g ${WARN} ${OPTIMIZE}
 
 # If libtheft is available, build additional property-based tests.
 # Uncomment these to use it in test_heatshrink_dynamic.
 #CFLAGS += -DHEATSHRINK_HAS_THEFT
-#LDFLAGS += -ltheft
+#THEFT_PATH=	/usr/local/
+#THEFT_INC=	-I${THEFT_PATH}/include/
+#LDFLAGS += -L${THEFT_PATH}/lib -ltheft
+
+CFLAGS += -std=c99 -g ${WARN} ${THEFT_INC} ${OPTIMIZE}
 
 all: heatshrink test_runners libraries
 
@@ -24,6 +27,8 @@ ci: test
 clean:
 	rm -f heatshrink test_heatshrink_{dynamic,static} \
 		*.o *.os *.od *.core *.a {dec,enc}_sm.png TAGS
+	rm -f ${BENCHMARK_OUT}/*
+	rmdir ${BENCHMARK_OUT}
 
 TAGS:
 	etags *.[ch]
@@ -36,6 +41,25 @@ dec_sm.png: dec_sm.dot
 enc_sm.png: enc_sm.dot
 	dot -o $@ -Tpng $<
 
+# Benchmarking
+CORPUS_ARCHIVE=	cantrbry.tar.gz
+CORPUS_URL=	http://corpus.canterbury.ac.nz/resources/${CORPUS_ARCHIVE}
+BENCHMARK_OUT=	benchmark_out
+
+## Uncomment one of these.
+DL=	curl -o ${CORPUS_ARCHIVE}
+#DL=	wget -O ${CORPUS_ARCHIVE}
+
+bench: heatshrink corpus
+	mkdir -p ${BENCHMARK_OUT}
+	time ./benchmark
+
+corpus: ${CORPUS_ARCHIVE}
+
+${CORPUS_ARCHIVE}:
+	${DL} ${CORPUS_URL}
+	cd ${BENCHMARK_OUT} && tar vzxf ../${CORPUS_ARCHIVE}
+
 # Installation
 PREFIX ?=	/usr/local
 INSTALL ?=	install

README.md

@@ -2,6 +2,7 @@
 
 A data compression/decompression library for embedded/real-time systems.
 
+
 ## Key Features:
 
 - **Low memory usage (as low as 50 bytes)**      
@@ -15,26 +16,113 @@ A data compression/decompression library for embedded/real-time systems.
 - **ISC license**      
     You can use it freely, even for commercial purposes.
 
+
 ## Getting Started:
 
 There is a standalone command-line program, `heatshrink`, but the
 encoder and decoder can also be used as libraries, independent of each
 other. To do so, copy `heatshrink_common.h`, `heatshrink_config.h`, and
 either `heatshrink_encoder.c` or `heatshrink_decoder.c` (and their
-respective header) into your project.
+respective header) into your project. For projects that use both,
+static libraries are built that use static and dynamic allocation.
 
 Dynamic allocation is used by default, but in an embedded context, you
 probably want to statically allocate the encoder/decoder. Set
 `HEATSHRINK_DYNAMIC_ALLOC` to 0 in `heatshrink_config.h`.
 
+
+### Basic Usage
+
+1. Allocate a `heatshrink_encoder` or `heatshrink_decoder` state machine
+using their `alloc` function, or statically allocate one and call their
+`_reset` function to initialize them. (See below for configuration
+options.)
+
+2. Use `sink` to sink an input buffer into the state machine. The
+`input_size` pointer argument will be set to indicate how many bytes of
+the input buffer were actually consumed. (If 0 bytes were conusmed, the
+buffer is full.)
+
+3. Use `poll` to move output from the state machine into an output
+buffer. The `output_size` pointer argument will be set to indicate how
+many bytes were output, and the function return value will indicate
+whether further output is available. (The state machine may not output
+any data until it has received enough input.)
+
+Repeat steps 2 and 3 to stream data through the state machine. Since
+it's doing data compression, the input and output sizes can vary
+significantly. Looping will be necessary to buffer the input and output
+as the data is processed.
+
+4. When the end of the input stream is reached, call `finish` to notify
+the state machine that no more input is available. The return value from
+`finish` will indicate whether any output remains. if so, call `poll` to
+get more.
+
+Continue calling `finish` and `poll`ing to flush remaining output until
+`finish` indicates that the output has been exhausted.
+
+Sinking more data after `finish` has been called will not work without
+calling `reset` on the state machine.
+
+
+## Configuration
+
+heatshrink has a couple configuration options, which impact its resource
+usage and how effectively it can compress data. These are set when
+dynamically allocating an encoder or decoder, or in `heatshrink_config.h`
+if they are statically allocated.
+
+- `window_sz2`, `-w` in the CLI: Set the window size to 2^W bytes.
+
+The window size determines how far back in the input can be searched for
+repeated patterns. A `window_sz2` of 8 will only use 256 bytes (2^8),
+while a `window_sz2` of 10 will use 1024 bytes (2^10). The latter uses
+more memory, but may also compress more effectively by detecting more
+repetition.
+
+The `window_sz2` setting currently must be between 4 and 15.
+
+- `lookahead_sz2`, `-l` in the CLI: Set the lookahead size to 2^L bytes.
+
+The lookahead size determines the max length for repeated patterns that
+are found. If the `lookahead_sz2` is 4, a 50-byte run of 'a' characters
+will be represented as several repeated 16-byte patterns (2^4 is 16),
+whereas a larger `lookahead_sz2` may be able to represent it all at
+once. The number of bits used for the lookahead size is fixed, so an
+overly large lookahead size can reduce compression by adding unused
+size bits to small patterns.
+
+The `lookahead_sz2` setting currently must be between 3 and the
+`window_sz2` - 1.
+
+- `input_buffer_size` - How large an input buffer to use for the
+decoder. This impacts how much work the decoder can do in a single
+step, and a larger buffer will use more memory. An extremely small
+buffer (say, 1 byte) will add overhead due to lots of suspend/resume
+function calls, but should not change how well data compresses.
+
+
+### Recommended Defaults
+
+For embedded/low memory contexts, a `window_sz2` in the 8 to 10 range is
+probably a good default, depending on how tight memory is. Smaller or
+larger window sizes may make better trade-offs in specific
+circumstances, but should be checked with representative data.
+
+The `lookahead_sz2` should probably start near the `window_sz2`/2, e.g.
+-w 8 -l 4 or -w 10 -l 5. The command-line program can be used to measure
+how well test data works with different settings.
+
+
 ## More Information and Benchmarks:
 
 heatshrink is based on [LZSS], since it's particularly suitable for
 compression in small amounts of memory. It can use an optional, small
 [index] to make compression significantly faster, but otherwise can run
 in under 100 bytes of memory. The index currently adds 2^(window size+1)
 bytes to memory usage for compression, and temporarily allocates 512
-bytes on the stack during index construction.
+bytes on the stack during index construction (if the index is enabled).
 
 For more information, see the [blog post] for an overview, and the
 `heatshrink_encoder.h` / `heatshrink_decoder.h` header files for API
@@ -44,6 +132,7 @@ documentation.
 [index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/
 [LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski
 
+
 ## Build Status
 
   [![Build Status](https://travis-ci.org/atomicobject/heatshrink.png)](http://travis-ci.org/atomicobject/heatshrink)

benchmark

@@ -0,0 +1,52 @@
+#!/bin/sh
+
+BENCHMARK_OUT=benchmark_out
+HS=../heatshrink
+
+cd ${BENCHMARK_OUT}
+
+# Files in the Canterbury Corpus
+# http://corpus.canterbury.ac.nz/resources/cantrbry.tar.gz
+FILES='alice29.txt
+asyoulik.txt
+cp.html
+fields.c
+grammar.lsp
+kennedy.xls
+lcet10.txt
+plrabn12.txt
+ptt5
+sum
+xargs.1'
+
+rm -f benchmark.output
+
+# Run several combinations of -w W -l L,
+# note compression ratios and check uncompressed output matches input
+for W in 6 7 8 9 10 11 12; do
+    for L in 5 6 7 8; do
+        if [ $L -lt $W ]; then
+            for f in ${FILES}
+            do 
+                IN_FILE="${f}"
+                COMPRESSED_FILE="${f}.hsz.${W}_${L}"
+                UNCOMPRESSED_FILE="${f}.orig.${W}_${L}"
+                ${HS} -e -v -w ${W} -l ${L} ${IN_FILE} ${COMPRESSED_FILE} >> benchmark.output
+                ${HS} -d -v -w ${W} -l ${L} ${COMPRESSED_FILE} ${UNCOMPRESSED_FILE} > /dev/null
+
+                if [ $(stat -f "%z" ${IN_FILE}) != $(stat -f "%z" ${UNCOMPRESSED_FILE}) ];
+                then
+                    printf "\n\n\nWARNING: size of %s does not match size of %s\n\n\n" \
+			${IN_FILE} ${UNCOMPRESSED_FILE}
+                else
+                    printf "pass: -w %2d -l %2d  %s\n" ${W} ${L} "${f}"
+                fi
+
+                rm ${COMPRESSED_FILE} ${UNCOMPRESSED_FILE}
+            done
+        fi
+    done
+done
+
+# Print totals and averages
+awk '{ t += $2; c++ }; END { printf("====\nTotal compression: %.02f%% for %d documents (avg. %0.2f%%)\n", t, c, t / c) }' benchmark.output

dec_sm.dot

@@ -1,27 +1,25 @@
 digraph {
         graph [label="Decoder state machine", labelloc="t"]
         Start [style="invis", shape="point"]
-        empty
-        input_available
+        tag_bit
         yield_literal
         backref_index_msb
         backref_index_lsb
         backref_count_msb
         backref_count_lsb
         yield_backref
-        check_for_more_input
         done [peripheries=2]
 
-        empty->input_available [label="sink()", color="blue", weight=10]
-        Start->empty
+        tag_bit->tag_bit [label="sink()", color="blue", weight=10]
+        Start->tag_bit
 
-        input_available->yield_literal [label="pop 1-bit"]
-        input_available->backref_index_msb [label="pop 0-bit", weight=10]
-        input_available->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10]
+        tag_bit->yield_literal [label="pop 1-bit"]
+        tag_bit->backref_index_msb [label="pop 0-bit", weight=10]
+        tag_bit->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10]
 
         yield_literal->yield_literal [label="sink()", color="blue"]
         yield_literal->yield_literal [label="poll()", color="red"]
-        yield_literal->check_for_more_input [label="poll(), done", color="red"]
+        yield_literal->tag_bit [label="poll(), done", color="red"]
 
         backref_index_msb->backref_index_msb [label="sink()", color="blue"]
         backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10]
@@ -42,11 +40,8 @@ digraph {
 
         yield_backref->yield_backref [label="sink()", color="blue"]
         yield_backref->yield_backref [label="poll()", color="red"]
-        yield_backref->check_for_more_input [label="poll(), done",
+        yield_backref->tag_bit [label="poll(), done",
             color="red", weight=10]
 
-        check_for_more_input->empty [label="no"]
-        check_for_more_input->input_available [label="yes"]
-
-        empty->done [label="finish()", color="blue"]
+        tag_bit->done [label="finish()", color="blue"]
 }