PR/573: Nick Terrell: Add zstd support

diff --git a/magic/Magdir/compress b/magic/Magdir/compress
index 23c67ae54268bcfb71eb271ef83536b3dbdf0fa8..8d790f7af0adb20fb59ecd66df36553c18bc086e 100644 (file)
--- a/magic/Magdir/compress
+++ b/magic/Magdir/compress
@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# $File: compress,v 1.64 2015/07/27 15:41:09 christos Exp $
+# $File: compress,v 1.65 2015/12/04 20:48:03 christos Exp $
 # compress:  file(1) magic for pure-compression formats (no archives)
 #
 # compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc.
@@ -220,6 +220,56 @@
 0      lelong          0x184c2102      LZ4 compressed data (v0.1-v0.9)
 !:mime application/x-lz4
 
+# Zstandard/LZ4 skippable frames
+# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
+0         lelong&0xFFFFFFF0  0x184D2A50
+>(4.l+8)  indirect
+
+# Zstandard Dictionary ID subroutine
+0     name        zstd-dictionary-id
+# Single Segment = True
+>0    byte        &0x20   \b, Dictionary ID:
+>>0   byte&0x03   0       None
+>>0   byte&0x03   1
+>>>1  byte        x       %u
+>>0   byte&0x03   2
+>>>1  leshort     x       %u
+>>0   byte&0x03   3
+>>>1  lelong      x       %u
+# Single Segment = False
+>0    byte        ^0x20   \b, Dictionary ID:
+>>0   byte&0x03   0       None
+>>0   byte&0x03   1
+>>>2  byte        x       %u
+>>0   byte&0x03   2
+>>>2  leshort     x       %u
+>>0   byte&0x03   3
+>>>2  lelong      x       %u
+
+# Zstandard compressed data
+# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
+0     lelong       0xFD2FB522  Zstandard compressed data (v0.2)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB523  Zstandard compressed data (v0.3)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB524  Zstandard compressed data (v0.4)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB525  Zstandard compressed data (v0.5)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB526  Zstandard compressed data (v0.6)
+!:mime  application/x-zstd
+0     lelong       0xFD2FB527  Zstandard compressed data (v0.7)
+!:mime  application/x-zstd
+>4    use          zstd-dictionary-id
+0     lelong       0xFD2FB528  Zstandard compressed data (v0.8+)
+!:mime  application/x-zstd
+>4    use          zstd-dictionary-id
+
+# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
+0  lelong    0xEC30A437  Zstandard dictionary
+!:mime  application/x-zstd-dictionary
+>4 lelong    x           (ID %u)
+
 # AFX compressed files (Wolfram Kleff)
 2      string          -afx-           AFX compressed file data
 

diff --git a/src/compress.c b/src/compress.c
index b19ffc0367dde541288ce558037ea2b1d9de27bc..cb4a85b300040b4927e277314d7ff903870fcc02 100644 (file)
--- a/src/compress.c
+++ b/src/compress.c
@@ -35,7 +35,7 @@
 #include "file.h"
 
 #ifndef lint
-FILE_RCSID("@(#)$File: compress.c,v 1.97 2016/05/13 23:02:28 christos Exp $")
+FILE_RCSID("@(#)$File: compress.c,v 1.98 2016/06/28 16:38:26 christos Exp $")
 #endif
 
 #include "magic.h"
@@ -132,6 +132,9 @@ static const char *lrzip_args[] = {
 static const char *lz4_args[] = {
        "lz4", "-cd", NULL
 };
+static const char *zstd_args[] = {
+       "zstd", "-cd", NULL
+};
 
 private const struct {
        const void *magic;
@@ -154,6 +157,7 @@ private const struct {
        { "\3757zXZ\0", 6, xz_args },           /* XZ Utils */
        { "LRZI",       4, lrzip_args },        /* LRZIP */
        { "\004\"M\030",4, lz4_args },          /* LZ4 */
+       { "\x28\xB5\x2F\xFD", 4, zstd_args },   /* zstd */
 #ifdef ZLIBSUPPORT
        { zlibcmp,      0, zlib_args },         /* zlib */
 #endif

diff --git a/tests/zstd-3-skippable-frames.result b/tests/zstd-3-skippable-frames.result
new file mode 100644 (file)
index 0000000..468070c
--- /dev/null
+++ b/tests/zstd-3-skippable-frames.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 1
\ No newline at end of file

diff --git a/tests/zstd-dictionary-0.result b/tests/zstd-dictionary-0.result
new file mode 100644 (file)
index 0000000..e1b9c62
--- /dev/null
+++ b/tests/zstd-dictionary-0.result
@@ -0,0 +1 @@
+Zstandard dictionary (ID 0)
\ No newline at end of file

diff --git a/tests/zstd-dictionary-1.result b/tests/zstd-dictionary-1.result
new file mode 100644 (file)
index 0000000..dd9f5f1
--- /dev/null
+++ b/tests/zstd-dictionary-1.result
@@ -0,0 +1 @@
+Zstandard dictionary (ID 1)
\ No newline at end of file

diff --git a/tests/zstd-dictionary-2.result b/tests/zstd-dictionary-2.result
new file mode 100644 (file)
index 0000000..e1ac5b3
--- /dev/null
+++ b/tests/zstd-dictionary-2.result
@@ -0,0 +1 @@
+Zstandard dictionary (ID 285212672)
\ No newline at end of file

diff --git a/tests/zstd-skippable-frame-0.result b/tests/zstd-skippable-frame-0.result
new file mode 100644 (file)
index 0000000..a4e3e48
--- /dev/null
+++ b/tests/zstd-skippable-frame-0.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.2)
\ No newline at end of file

diff --git a/tests/zstd-skippable-frame-4.result b/tests/zstd-skippable-frame-4.result
new file mode 100644 (file)
index 0000000..cecb5e7
--- /dev/null
+++ b/tests/zstd-skippable-frame-4.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.3)
\ No newline at end of file

diff --git a/tests/zstd-skippable-frame-8.result b/tests/zstd-skippable-frame-8.result
new file mode 100644 (file)
index 0000000..528f701
--- /dev/null
+++ b/tests/zstd-skippable-frame-8.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.4)
\ No newline at end of file

diff --git a/tests/zstd-skippable-frame-C.result b/tests/zstd-skippable-frame-C.result
new file mode 100644 (file)
index 0000000..468070c
--- /dev/null
+++ b/tests/zstd-skippable-frame-C.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 1
\ No newline at end of file

diff --git a/tests/zstd-v0.2-FF.result b/tests/zstd-v0.2-FF.result
new file mode 100644 (file)
index 0000000..a4e3e48
--- /dev/null
+++ b/tests/zstd-v0.2-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.2)
\ No newline at end of file

diff --git a/tests/zstd-v0.2-FF.testfile b/tests/zstd-v0.2-FF.testfile
new file mode 100644 (file)
index 0000000..6fe4f27
--- /dev/null
+++ b/tests/zstd-v0.2-FF.testfile
@@ -0,0 +1 @@
+"µ/ýÿ\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.3-FF.result b/tests/zstd-v0.3-FF.result
new file mode 100644 (file)
index 0000000..cecb5e7
--- /dev/null
+++ b/tests/zstd-v0.3-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.3)
\ No newline at end of file

diff --git a/tests/zstd-v0.3-FF.testfile b/tests/zstd-v0.3-FF.testfile
new file mode 100644 (file)
index 0000000..dc50469
--- /dev/null
+++ b/tests/zstd-v0.3-FF.testfile
@@ -0,0 +1 @@
+#µ/ýÿ\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.4-FF.result b/tests/zstd-v0.4-FF.result
new file mode 100644 (file)
index 0000000..528f701
--- /dev/null
+++ b/tests/zstd-v0.4-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.4)
\ No newline at end of file

diff --git a/tests/zstd-v0.4-FF.testfile b/tests/zstd-v0.4-FF.testfile
new file mode 100644 (file)
index 0000000..f2768a2
--- /dev/null
+++ b/tests/zstd-v0.4-FF.testfile
@@ -0,0 +1 @@
+$µ/ýÿ\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.5-FF.result b/tests/zstd-v0.5-FF.result
new file mode 100644 (file)
index 0000000..815c80d
--- /dev/null
+++ b/tests/zstd-v0.5-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.5)
\ No newline at end of file

diff --git a/tests/zstd-v0.5-FF.testfile b/tests/zstd-v0.5-FF.testfile
new file mode 100644 (file)
index 0000000..a25f337
--- /dev/null
+++ b/tests/zstd-v0.5-FF.testfile
@@ -0,0 +1 @@
+%µ/ýÿ\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.6-FF.result b/tests/zstd-v0.6-FF.result
new file mode 100644 (file)
index 0000000..b28af6b
--- /dev/null
+++ b/tests/zstd-v0.6-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.6)
\ No newline at end of file

diff --git a/tests/zstd-v0.6-FF.testfile b/tests/zstd-v0.6-FF.testfile
new file mode 100644 (file)
index 0000000..1c8ca59
--- /dev/null
+++ b/tests/zstd-v0.6-FF.testfile
@@ -0,0 +1 @@
+&µ/ýÿ\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.7-00.result b/tests/zstd-v0.7-00.result
new file mode 100644 (file)
index 0000000..72e0e27
--- /dev/null
+++ b/tests/zstd-v0.7-00.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.7), Dictionary ID: None
\ No newline at end of file

diff --git a/tests/zstd-v0.7-21.result b/tests/zstd-v0.7-21.result
new file mode 100644 (file)
index 0000000..a4d43c8
--- /dev/null
+++ b/tests/zstd-v0.7-21.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.7), Dictionary ID: 1
\ No newline at end of file

diff --git a/tests/zstd-v0.7-21.testfile b/tests/zstd-v0.7-21.testfile
new file mode 100644 (file)
index 0000000..b40294e
--- /dev/null
+++ b/tests/zstd-v0.7-21.testfile
@@ -0,0 +1 @@
+'µ/ý!\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.7-22.result b/tests/zstd-v0.7-22.result
new file mode 100644 (file)
index 0000000..b4336c9
--- /dev/null
+++ b/tests/zstd-v0.7-22.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.7), Dictionary ID: 513
\ No newline at end of file

diff --git a/tests/zstd-v0.7-22.testfile b/tests/zstd-v0.7-22.testfile
new file mode 100644 (file)
index 0000000..8b72d68
--- /dev/null
+++ b/tests/zstd-v0.7-22.testfile
@@ -0,0 +1 @@
+'µ/ý"\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-00.result b/tests/zstd-v0.8-00.result
new file mode 100644 (file)
index 0000000..67edebd
--- /dev/null
+++ b/tests/zstd-v0.8-00.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: None
\ No newline at end of file

diff --git a/tests/zstd-v0.8-01.result b/tests/zstd-v0.8-01.result
new file mode 100644 (file)
index 0000000..a1239a7
--- /dev/null
+++ b/tests/zstd-v0.8-01.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 2
\ No newline at end of file

diff --git a/tests/zstd-v0.8-01.testfile b/tests/zstd-v0.8-01.testfile
new file mode 100644 (file)
index 0000000..88735e4
--- /dev/null
+++ b/tests/zstd-v0.8-01.testfile
@@ -0,0 +1 @@
+(µ/ý\ 1\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-02.result b/tests/zstd-v0.8-02.result
new file mode 100644 (file)
index 0000000..82beab8
--- /dev/null
+++ b/tests/zstd-v0.8-02.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 770
\ No newline at end of file

diff --git a/tests/zstd-v0.8-02.testfile b/tests/zstd-v0.8-02.testfile
new file mode 100644 (file)
index 0000000..db55433
--- /dev/null
+++ b/tests/zstd-v0.8-02.testfile
@@ -0,0 +1 @@
+(µ/ý\ 2\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-03.result b/tests/zstd-v0.8-03.result
new file mode 100644 (file)
index 0000000..6aaaa19
--- /dev/null
+++ b/tests/zstd-v0.8-03.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 84148994
\ No newline at end of file

diff --git a/tests/zstd-v0.8-03.testfile b/tests/zstd-v0.8-03.testfile
new file mode 100644 (file)
index 0000000..506b344
--- /dev/null
+++ b/tests/zstd-v0.8-03.testfile
@@ -0,0 +1 @@
+(µ/ý\ 3\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-16.result b/tests/zstd-v0.8-16.result
new file mode 100644 (file)
index 0000000..82beab8
--- /dev/null
+++ b/tests/zstd-v0.8-16.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 770
\ No newline at end of file

diff --git a/tests/zstd-v0.8-16.testfile b/tests/zstd-v0.8-16.testfile
new file mode 100644 (file)
index 0000000..3f87f79
--- /dev/null
+++ b/tests/zstd-v0.8-16.testfile
@@ -0,0 +1 @@
+(µ/ý\16\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-20.result b/tests/zstd-v0.8-20.result
new file mode 100644 (file)
index 0000000..67edebd
--- /dev/null
+++ b/tests/zstd-v0.8-20.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: None
\ No newline at end of file

diff --git a/tests/zstd-v0.8-20.testfile b/tests/zstd-v0.8-20.testfile
new file mode 100644 (file)
index 0000000..76fdbb8
--- /dev/null
+++ b/tests/zstd-v0.8-20.testfile
@@ -0,0 +1 @@
+(µ/ý \ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-21.result b/tests/zstd-v0.8-21.result
new file mode 100644 (file)
index 0000000..468070c
--- /dev/null
+++ b/tests/zstd-v0.8-21.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 1
\ No newline at end of file

diff --git a/tests/zstd-v0.8-21.testfile b/tests/zstd-v0.8-21.testfile
new file mode 100644 (file)
index 0000000..9ebeff4
--- /dev/null
+++ b/tests/zstd-v0.8-21.testfile
@@ -0,0 +1 @@
+(µ/ý!\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-22.result b/tests/zstd-v0.8-22.result
new file mode 100644 (file)
index 0000000..9869d24
--- /dev/null
+++ b/tests/zstd-v0.8-22.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 513
\ No newline at end of file

diff --git a/tests/zstd-v0.8-22.testfile b/tests/zstd-v0.8-22.testfile
new file mode 100644 (file)
index 0000000..f2e55bf
--- /dev/null
+++ b/tests/zstd-v0.8-22.testfile
@@ -0,0 +1 @@
+(µ/ý"\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-23.result b/tests/zstd-v0.8-23.result
new file mode 100644 (file)
index 0000000..d38420f
--- /dev/null
+++ b/tests/zstd-v0.8-23.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 67305985
\ No newline at end of file

diff --git a/tests/zstd-v0.8-23.testfile b/tests/zstd-v0.8-23.testfile
new file mode 100644 (file)
index 0000000..f66a18f
--- /dev/null
+++ b/tests/zstd-v0.8-23.testfile
@@ -0,0 +1 @@
+(µ/ý#\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-F4.result b/tests/zstd-v0.8-F4.result
new file mode 100644 (file)
index 0000000..67edebd
--- /dev/null
+++ b/tests/zstd-v0.8-F4.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: None
\ No newline at end of file

diff --git a/tests/zstd-v0.8-F4.testfile b/tests/zstd-v0.8-F4.testfile
new file mode 100644 (file)
index 0000000..a4e4240
--- /dev/null
+++ b/tests/zstd-v0.8-F4.testfile
@@ -0,0 +1 @@
+(µ/ýô\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file

diff --git a/tests/zstd-v0.8-FF.result b/tests/zstd-v0.8-FF.result
new file mode 100644 (file)
index 0000000..d38420f
--- /dev/null
+++ b/tests/zstd-v0.8-FF.result
@@ -0,0 +1 @@
+Zstandard compressed data (v0.8+), Dictionary ID: 67305985
\ No newline at end of file

diff --git a/tests/zstd-v0.8-FF.testfile b/tests/zstd-v0.8-FF.testfile
new file mode 100644 (file)
index 0000000..bc63911
--- /dev/null
+++ b/tests/zstd-v0.8-FF.testfile
@@ -0,0 +1 @@
+(µ/ýÿ\ 1\ 2\ 3\ 4\ 5
\ No newline at end of file