mirror of
https://github.com/freebsd/freebsd-src.git
synced 2026-10-28 11:25:21 +01:00
MFV: zstd: Remove entries listed in FREEBSD-Xlist
This commit is contained in:
Xin LI committed
commit
111a42d7d7
50 files changed
-14007
No files matched your search
@@ -1,9 +0,0 @@
|
||||
[cxx]
|
||||
cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
|
||||
cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
|
||||
cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
|
||||
cxxflags = -std=c++11 -Wno-deprecated-declarations
|
||||
gtest_dep = //contrib/pzstd:gtest
|
||||
|
||||
[httpserver]
|
||||
port = 0
|
||||
@@ -1 +0,0 @@
|
||||
c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b
|
||||
@@ -1,489 +0,0 @@
|
||||
# Contributing to Zstandard
|
||||
We want to make contributing to this project as easy and transparent as
|
||||
possible.
|
||||
|
||||
## Our Development Process
|
||||
New versions are being developed in the "dev" branch,
|
||||
or in their own feature branch.
|
||||
When they are deemed ready for a release, they are merged into "release".
|
||||
|
||||
As a consequence, all contributions must stage first through "dev"
|
||||
or their own feature branch.
|
||||
|
||||
## Pull Requests
|
||||
We actively welcome your pull requests.
|
||||
|
||||
1. Fork the repo and create your branch from `dev`.
|
||||
2. If you've added code that should be tested, add tests.
|
||||
3. If you've changed APIs, update the documentation.
|
||||
4. Ensure the test suite passes.
|
||||
5. Make sure your code lints.
|
||||
6. If you haven't already, complete the Contributor License Agreement ("CLA").
|
||||
|
||||
## Contributor License Agreement ("CLA")
|
||||
In order to accept your pull request, we need you to submit a CLA. You only need
|
||||
to do this once to work on any of Facebook's open source projects.
|
||||
|
||||
Complete your CLA here: <https://code.facebook.com/cla>
|
||||
|
||||
## Workflow
|
||||
Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd
|
||||
will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple
|
||||
related changes into a branch.
|
||||
|
||||
Our contribution process works in three main stages:
|
||||
1. Local development
|
||||
* Update:
|
||||
* Checkout your fork of zstd if you have not already
|
||||
```
|
||||
git checkout https://github.com/<username>/zstd
|
||||
cd zstd
|
||||
```
|
||||
* Update your local dev branch
|
||||
```
|
||||
git pull https://github.com/facebook/zstd dev
|
||||
git push origin dev
|
||||
```
|
||||
* Topic and development:
|
||||
* Make a new branch on your fork about the topic you're developing for
|
||||
```
|
||||
# branch names should be concise but sufficiently informative
|
||||
git checkout -b <branch-name>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Make commits and push
|
||||
```
|
||||
# make some changes =
|
||||
git add -u && git commit -m <message>
|
||||
git push origin <branch-name>
|
||||
```
|
||||
* Note: run local tests to ensure that your changes didn't break existing functionality
|
||||
* Quick check
|
||||
```
|
||||
make check
|
||||
```
|
||||
* Longer check
|
||||
```
|
||||
make test
|
||||
```
|
||||
2. Code Review and CI tests
|
||||
* Ensure CI tests pass:
|
||||
* Before sharing anything to the community, create a pull request in your own fork against the dev branch
|
||||
and make sure that all GitHub Actions CI tests pass. See the Continuous Integration section below for more information.
|
||||
* Ensure that static analysis passes on your development machine. See the Static Analysis section
|
||||
below to see how to do this.
|
||||
* Create a pull request:
|
||||
* When you are ready to share you changes to the community, create a pull request from your branch
|
||||
to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home
|
||||
page.
|
||||
* From there, select the branch where you made changes as your source branch and facebook:dev
|
||||
as the destination.
|
||||
* Examine the diff presented between the two branches to make sure there is nothing unexpected.
|
||||
* Write a good pull request description:
|
||||
* While there is no strict template that our contributors follow, we would like them to
|
||||
sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests,
|
||||
at least indirectly, address the following points.
|
||||
* Is this pull request important and why?
|
||||
* Is it addressing an issue? If so, what issue? (provide links for convenience please)
|
||||
* Is this a new feature? If so, why is it useful and/or necessary?
|
||||
* Are there background references and documents that reviewers should be aware of to properly assess this change?
|
||||
* Note: make sure to point out any design and architectural decisions that you made and the rationale behind them.
|
||||
* Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>)
|
||||
* Submit the pull request and iterate with feedback.
|
||||
3. Merge and Release
|
||||
* Getting approval:
|
||||
* You will have to iterate on your changes with feedback from other collaborators to reach a point
|
||||
where your pull request can be safely merged.
|
||||
* To avoid too many comments on style and convention, make sure that you have a
|
||||
look at our style section below before creating a pull request.
|
||||
* Eventually, someone from the zstd team will approve your pull request and not long after merge it into
|
||||
the dev branch.
|
||||
* Housekeeping:
|
||||
* Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure
|
||||
the corresponding issue is mentioned. If your change 'fixes' or completely addresses the
|
||||
issue at hand, then please indicate this by requesting that an issue be closed by commenting.
|
||||
* Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
|
||||
that the change must make it to an official zstd release for it to be meaningful. We recommend
|
||||
that contributors track the activity on their pull request and corresponding issue(s) page(s) until
|
||||
their change makes it to the next release of zstd. Users will often discover bugs in your code or
|
||||
suggest ways to refine and improve your initial changes even after the pull request is merged.
|
||||
|
||||
## Static Analysis
|
||||
Static analysis is a process for examining the correctness or validity of a program without actually
|
||||
executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for
|
||||
static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build.
|
||||
|
||||
Once installed, you can ensure that our static analysis tests pass on your local development machine
|
||||
by running:
|
||||
```
|
||||
make staticAnalyze
|
||||
```
|
||||
|
||||
In general, you can use `scan-build` to static analyze any build script. For example, to static analyze
|
||||
just `contrib/largeNbDicts` and nothing else, you can run:
|
||||
|
||||
```
|
||||
scan-build make -C contrib/largeNbDicts largeNbDicts
|
||||
```
|
||||
|
||||
### Pitfalls of static analysis
|
||||
`scan-build` is part of our regular CI suite. Other static analyzers are not.
|
||||
|
||||
It can be useful to look at additional static analyzers once in a while (and we do), but it's not a good idea to multiply the nb of analyzers run continuously at each commit and PR. The reasons are :
|
||||
|
||||
- Static analyzers are full of false positive. The signal to noise ratio is actually pretty low.
|
||||
- A good CI policy is "zero-warning tolerance". That means that all issues must be solved, including false positives. This quickly becomes a tedious workload.
|
||||
- Multiple static analyzers will feature multiple kind of false positives, sometimes applying to the same code but in different ways leading to :
|
||||
+ tortuous code, trying to please multiple constraints, hurting readability and therefore maintenance. Sometimes, such complexity introduce other more subtle bugs, that are just out of scope of the analyzers.
|
||||
+ sometimes, these constraints are mutually exclusive : if one try to solve one, the other static analyzer will complain, they can't be both happy at the same time.
|
||||
- As if that was not enough, the list of false positives change with each version. It's hard enough to follow one static analyzer, but multiple ones with their own update agenda, this quickly becomes a massive velocity reducer.
|
||||
|
||||
This is different from running a static analyzer once in a while, looking at the output, and __cherry picking__ a few warnings that seem helpful, either because they detected a genuine risk of bug, or because it helps expressing the code in a way which is more readable or more difficult to misuse. These kinds of reports can be useful, and are accepted.
|
||||
|
||||
## Continuous Integration
|
||||
CI tests run every time a pull request (PR) is created or updated. The exact tests
|
||||
that get run will depend on the destination branch you specify. Some tests take
|
||||
longer to run than others. Currently, our CI is set up to run a short
|
||||
series of tests when creating a PR to the dev branch and a longer series of tests
|
||||
when creating a PR to the release branch. You can look in the configuration files
|
||||
of the respective CI platform for more information on what gets run when.
|
||||
|
||||
Most people will just want to create a PR with the destination set to their local dev
|
||||
branch of zstd. You can then find the status of the tests on the PR's page. You can also
|
||||
re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
|
||||
|
||||
Almost all of zstd's CI runs on GitHub Actions (configured at `.github/workflows`), which will automatically run on PRs to your
|
||||
own fork. A small number of tests run on other services (e.g. Travis CI, Circle CI, Appveyor).
|
||||
These require work to set up on your local fork, and (at least for Travis CI) cost money.
|
||||
Therefore, if the PR on your local fork passes GitHub Actions, feel free to submit a PR
|
||||
against the main repo.
|
||||
|
||||
### Third-party CI
|
||||
A small number of tests cannot run on GitHub Actions, or have yet to be migrated.
|
||||
For these, we use a variety of third-party services (listed below). It is not necessary to set
|
||||
these up on your fork in order to contribute to zstd; however, we do link to instructions for those
|
||||
who want earlier signal.
|
||||
|
||||
| Service | Purpose | Setup Links | Config Path |
|
||||
|-----------|------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
|
||||
| Travis CI | Used for testing on non-x86 architectures such as PowerPC | https://docs.travis-ci.com/user/tutorial/#to-get-started-with-travis-ci-using-github <br> https://github.com/marketplace/travis-ci | `.travis.yml` |
|
||||
| AppVeyor | Used for some Windows testing (e.g. cygwin, mingw) | https://www.appveyor.com/blog/2018/10/02/github-apps-integration/ <br> https://github.com/marketplace/appveyor | `appveyor.yml` |
|
||||
| Cirrus CI | Used for testing on FreeBSD | https://github.com/marketplace/cirrus-ci/ | `.cirrus.yml` |
|
||||
| Circle CI | Historically was used to provide faster signal,<br/> but we may be able to migrate these to Github Actions | https://circleci.com/docs/2.0/getting-started/#setting-up-circleci <br> https://youtu.be/Js3hMUsSZ2c <br> https://circleci.com/docs/2.0/enable-checks/ | `.circleci/config.yml` |
|
||||
|
||||
Note: the instructions linked above mostly cover how to set up a repository with CI from scratch.
|
||||
The general idea should be the same for setting up CI on your fork of zstd, but you may have to
|
||||
follow slightly different steps. In particular, please ignore any instructions related to setting up
|
||||
config files (since zstd already has configs for each of these services).
|
||||
|
||||
## Performance
|
||||
Performance is extremely important for zstd and we only merge pull requests whose performance
|
||||
landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
|
||||
This high bar for performance means that every PR which has the potential to
|
||||
impact performance takes a very long time for us to properly review. That being said, we
|
||||
always welcome contributions to improve performance (or worsen performance for the trade-off of
|
||||
something else). Please keep the following in mind before submitting a performance related PR:
|
||||
|
||||
1. Zstd isn't as old as gzip but it has been around for time now and its evolution is
|
||||
very well documented via past Github issues and pull requests. It may be the case that your
|
||||
particular performance optimization has already been considered in the past. Please take some
|
||||
time to search through old issues and pull requests using keywords specific to your
|
||||
would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected
|
||||
on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case,
|
||||
it will be helpful for you to have context from that topic's history before contributing.
|
||||
2. The distinction between noise and actual performance gains can unfortunately be very subtle
|
||||
especially when microbenchmarking extremely small wins or losses. The only remedy to getting
|
||||
something subtle merged is extensive benchmarking. You will be doing us a great favor if you
|
||||
take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc)
|
||||
benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark
|
||||
your changes on every single processor and os out there (and neither will we) but do that best
|
||||
you can:) We've added some things to think about when benchmarking below in the Benchmarking
|
||||
Performance section which might be helpful for you.
|
||||
3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly
|
||||
legitimate thing to do as long as it does not harm the overall performance health of Zstd.
|
||||
This is a hard balance to strike but please keep in mind other aspects of Zstd when
|
||||
submitting changes that are clang-specific, windows-specific, etc.
|
||||
|
||||
## Benchmarking Performance
|
||||
Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical
|
||||
testing over theoretical speculation. This guide it not perfect but for most scenarios, it
|
||||
is a good place to start.
|
||||
|
||||
### Stability
|
||||
Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable
|
||||
benchmarking machine. A virtual machine, a machine with shared resources, or your laptop
|
||||
will typically not be stable enough to obtain reliable benchmark results. If you can get your
|
||||
hands on a desktop, this is usually a better scenario.
|
||||
|
||||
Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
|
||||
do a little more work to ensure that you are in fact measuring the changes you've made and not
|
||||
noise. Here are some things you can do to make your benchmarks more stable:
|
||||
|
||||
1. The most simple thing you can do to drastically improve the stability of your benchmark is
|
||||
to run it multiple times and then aggregate the results of those runs. As a general rule of
|
||||
thumb, the smaller the change you are trying to measure, the more samples of benchmark runs
|
||||
you will have to aggregate over to get reliable results. Here are some additional things to keep in
|
||||
mind when running multiple trials:
|
||||
* How you aggregate your samples are important. You might be tempted to use the mean of your
|
||||
results. While this is certainly going to be a more stable number than a raw single sample
|
||||
benchmark number, you might have more luck by taking the median. The mean is not robust to
|
||||
outliers whereas the median is. Better still, you could simply take the fastest speed your
|
||||
benchmark achieved on each run since that is likely the fastest your process will be
|
||||
capable of running your code. In our experience, this (aggregating by just taking the sample
|
||||
with the fastest running time) has been the most stable approach.
|
||||
* The more samples you have, the more stable your benchmarks should be. You can verify
|
||||
your improved stability by looking at the size of your confidence intervals as you
|
||||
increase your sample count. These should get smaller and smaller. Eventually hopefully
|
||||
smaller than the performance win you are expecting.
|
||||
* Most processors will take some time to get `hot` when running anything. The observations
|
||||
you collect during that time period will very different from the true performance number. Having
|
||||
a very large number of sample will help alleviate this problem slightly but you can also
|
||||
address is directly by simply not including the first `n` iterations of your benchmark in
|
||||
your aggregations. You can determine `n` by simply looking at the results from each iteration
|
||||
and then hand picking a good threshold after which the variance in results seems to stabilize.
|
||||
2. You cannot really get reliable benchmarks if your host machine is simultaneously running
|
||||
another cpu/memory-intensive application in the background. If you are running benchmarks on your
|
||||
personal laptop for instance, you should close all applications (including your code editor and
|
||||
browser) before running your benchmarks. You might also have invisible background applications
|
||||
running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager
|
||||
on Windows. You will get more stable benchmark results of you end those processes as well.
|
||||
* If you have multiple cores, you can even run your benchmark on a reserved core to prevent
|
||||
pollution from other OS and user processes. There are a number of ways to do this depending
|
||||
on your OS:
|
||||
* On linux boxes, you have use https://github.com/lpechacek/cpuset.
|
||||
* On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows
|
||||
* On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2
|
||||
3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd.
|
||||
Dynamically linking your library will introduce some added variation (not a large amount but
|
||||
definitely some). Statically linking libzstd will be more stable. Static libraries should
|
||||
be enabled by default when building zstd.
|
||||
4. Use a profiler with a good high resolution timer. See the section below on profiling for
|
||||
details on this.
|
||||
5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS)
|
||||
6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on
|
||||
tmpfs avoids touching a real storage system, which can have a pretty big variability.
|
||||
|
||||
Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html
|
||||
|
||||
### Zstd benchmark
|
||||
The fastest signal you can get regarding your performance changes is via the in-build zstd cli
|
||||
bench option. You can run Zstd as you typically would for your scenario using some set of options
|
||||
and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline
|
||||
for that options you have just provided. If you want to look at the internals of how this
|
||||
benchmarking script works, you can check out programs/benchzstd.c
|
||||
|
||||
For example: say you have made a change that you believe improves the speed of zstd level 1. The
|
||||
very first thing you should use to assess whether you actually achieved any sort of improvement
|
||||
is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to
|
||||
specify a running time for your benchmark in seconds (default is 3 seconds).
|
||||
Usually, the longer the running time, the more stable your results will be.
|
||||
|
||||
```
|
||||
$ git checkout <commit-before-your-change>
|
||||
$ make && cp zstd zstd-old
|
||||
$ git checkout <commit-after-your-change>
|
||||
$ make && cp zstd zstd-new
|
||||
$ zstd-old -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.6 MB/s , 626.4 MB/s
|
||||
$ zstd-new -i5 -b1 <your-test-data>
|
||||
1<your-test-data> : 8990 -> 3992 (2.252), 302.8 MB/s , 628.4 MB/s
|
||||
```
|
||||
|
||||
Unless your performance win is large enough to be visible despite the intrinsic noise
|
||||
on your computer, benchzstd alone will likely not be enough to validate the impact of your
|
||||
changes. For example, the results of the example above indicate that effectively nothing
|
||||
changed but there could be a small <3% improvement that the noise on the host machine
|
||||
obscured. So unless you see a large performance win (10-15% consistently) using just
|
||||
this method of evaluation will not be sufficient.
|
||||
|
||||
### Profiling
|
||||
There are a number of great profilers out there. We're going to briefly mention how you can
|
||||
profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler`
|
||||
on Windows.
|
||||
|
||||
Say you have an idea for a change that you think will provide some good performance gains
|
||||
for level 1 compression on Zstd. Typically this means, you have identified a section of
|
||||
code that you think can be made to run faster.
|
||||
|
||||
The first thing you will want to do is make sure that the piece of code is actually taking up
|
||||
a notable amount of time to run. It is usually not worth optimizing something which accounts for less than
|
||||
0.0001% of the total running time. Luckily, there are tools to help with this.
|
||||
Profilers will let you see how much time your code spends inside a particular function.
|
||||
If your target code snippet is only part of a function, it might be worth trying to
|
||||
isolate that snippet by moving it to its own function (this is usually not necessary but
|
||||
might be).
|
||||
|
||||
Most profilers (including the profilers discussed below) will generate a call graph of
|
||||
functions for you. Your goal will be to find your function of interest in this call graph
|
||||
and then inspect the time spent inside of it. You might also want to look at the annotated
|
||||
assembly which most profilers will provide you with.
|
||||
|
||||
#### Instruments
|
||||
We will once again consider the scenario where you think you've identified a piece of code
|
||||
whose performance can be improved upon. Follow these steps to profile your code using
|
||||
Instruments.
|
||||
|
||||
1. Open Instruments
|
||||
2. Select `Time Profiler` from the list of standard templates
|
||||
3. Close all other applications except for your instruments window and your terminal
|
||||
4. Run your benchmarking script from your terminal window
|
||||
* You will want a benchmark that runs for at least a few seconds (5 seconds will
|
||||
usually be long enough). This way the profiler will have something to work with
|
||||
and you will have ample time to attach your profiler to this process:)
|
||||
* I will just use benchzstd as my benchmarmking script for this example:
|
||||
```
|
||||
$ zstd -b1 -i5 <my-data> # this will run for 5 seconds
|
||||
```
|
||||
5. Once you run your benchmarking script, switch back over to instruments and attach your
|
||||
process to the time profiler. You can do this by:
|
||||
* Clicking on the `All Processes` drop down in the top left of the toolbar.
|
||||
* Selecting your process from the dropdown. In my case, it is just going to be labeled
|
||||
`zstd`
|
||||
* Hitting the bright red record circle button on the top left of the toolbar
|
||||
6. You profiler will now start collecting metrics from your benchmarking script. Once
|
||||
you think you have collected enough samples (usually this is the case after 3 seconds of
|
||||
recording), stop your profiler.
|
||||
7. Make sure that in toolbar of the bottom window, `profile` is selected.
|
||||
8. You should be able to see your call graph.
|
||||
* If you don't see the call graph or an incomplete call graph, make sure you have compiled
|
||||
zstd and your benchmarking script using debug flags. On mac and linux, this just means
|
||||
you will have to supply the `-g` flag alone with your build script. You might also
|
||||
have to provide the `-fno-omit-frame-pointer` flag
|
||||
9. Dig down the graph to find your function call and then inspect it by double clicking
|
||||
the list item. You will be able to see the annotated source code and the assembly side by
|
||||
side.
|
||||
|
||||
#### Perf
|
||||
|
||||
This wiki has a pretty detailed tutorial on getting started working with perf so we'll
|
||||
leave you to check that out of you're getting started:
|
||||
|
||||
https://perf.wiki.kernel.org/index.php/Tutorial
|
||||
|
||||
Some general notes on perf:
|
||||
* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and
|
||||
counter statistics. Perf uses a high resolution timer and this is likely one
|
||||
of the first things your team will run when assessing your PR.
|
||||
* Perf has a long list of hardware counters that can be viewed with `perf --list`.
|
||||
When measuring optimizations, something worth trying is to make sure the hardware
|
||||
counters you expect to be impacted by your change are in fact being so. For example,
|
||||
if you expect the L1 cache misses to decrease with your change, you can look at the
|
||||
counter `L1-dcache-load-misses`
|
||||
* Perf hardware counters will not work on a virtual machine.
|
||||
|
||||
#### Visual Studio
|
||||
|
||||
TODO
|
||||
|
||||
## Issues
|
||||
We use GitHub issues to track public bugs. Please ensure your description is
|
||||
clear and has sufficient instructions to be able to reproduce the issue.
|
||||
|
||||
Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
|
||||
disclosure of security bugs. In those cases, please go through the process
|
||||
outlined on that page and do not file a public issue.
|
||||
|
||||
## Coding Style
|
||||
It's a pretty long topic, which is difficult to summarize in a single paragraph.
|
||||
As a rule of thumbs, try to imitate the coding style of
|
||||
similar lines of codes around your contribution.
|
||||
The following is a non-exhaustive list of rules employed in zstd code base:
|
||||
|
||||
### C90
|
||||
This code base is following strict C90 standard,
|
||||
with 2 extensions : 64-bit `long long` types, and variadic macros.
|
||||
This rule is applied strictly to code within `lib/` and `programs/`.
|
||||
Sub-project in `contrib/` are allowed to use other conventions.
|
||||
|
||||
### C++ direct compatibility : symbol mangling
|
||||
All public symbol declarations must be wrapped in `extern “C” { … }`,
|
||||
so that this project can be compiled as C++98 code,
|
||||
and linked into C++ applications.
|
||||
|
||||
### Minimal Frugal
|
||||
This design requirement is fundamental to preserve the portability of the code base.
|
||||
#### Dependencies
|
||||
- Reduce dependencies to the minimum possible level.
|
||||
Any dependency should be considered “bad” by default,
|
||||
and only tolerated because it provides a service in a better way than can be achieved locally.
|
||||
The only external dependencies this repository tolerates are
|
||||
standard C libraries, and in rare cases, system level headers.
|
||||
- Within `lib/`, this policy is even more drastic.
|
||||
The only external dependencies allowed are `<assert.h>`, `<stdlib.h>`, `<string.h>`,
|
||||
and even then, not directly.
|
||||
In particular, no function shall ever allocate on heap directly,
|
||||
and must use instead `ZSTD_malloc()` and equivalent.
|
||||
Other accepted non-symbol headers are `<stddef.h>` and `<limits.h>`.
|
||||
- Within the project, there is a strict hierarchy of dependencies that must be respected.
|
||||
`programs/` is allowed to depend on `lib/`, but only its public API.
|
||||
Within `lib/`, `lib/common` doesn't depend on any other directory.
|
||||
`lib/compress` and `lib/decompress` shall not depend on each other.
|
||||
`lib/dictBuilder` can depend on `lib/common` and `lib/compress`, but not `lib/decompress`.
|
||||
#### Resources
|
||||
- Functions in `lib/` must use very little stack space,
|
||||
several dozens of bytes max.
|
||||
Everything larger must use the heap allocator,
|
||||
or require a scratch buffer to be emplaced manually.
|
||||
|
||||
### Naming
|
||||
* All public symbols are prefixed with `ZSTD_`
|
||||
+ private symbols, with a scope limited to their own unit, are free of this restriction.
|
||||
However, since `libzstd` source code can be amalgamated,
|
||||
each symbol name must attempt to be (and remain) unique.
|
||||
Avoid too generic names that could become ground for future collisions.
|
||||
This generally implies usage of some form of prefix.
|
||||
* For symbols (functions and variables), naming convention is `PREFIX_camelCase`.
|
||||
+ In some advanced cases, one can also find :
|
||||
- `PREFIX_prefix2_camelCase`
|
||||
- `PREFIX_camelCase_extendedQualifier`
|
||||
* Multi-words names generally consist of an action followed by object:
|
||||
- for example : `ZSTD_createCCtx()`
|
||||
* Prefer positive actions
|
||||
- `goBackward` rather than `notGoForward`
|
||||
* Type names (`struct`, etc.) follow similar convention,
|
||||
except that they are allowed and even invited to start by an Uppercase letter.
|
||||
Example : `ZSTD_CCtx`, `ZSTD_CDict`
|
||||
* Macro names are all Capital letters.
|
||||
The same composition rules (`PREFIX_NAME_QUALIFIER`) apply.
|
||||
* File names are all lowercase letters.
|
||||
The convention is `snake_case`.
|
||||
File names **must** be unique across the entire code base,
|
||||
even when they stand in clearly separated directories.
|
||||
|
||||
### Qualifiers
|
||||
* This code base is `const` friendly, if not `const` fanatical.
|
||||
Any variable that can be `const` (aka. read-only) **must** be `const`.
|
||||
Any pointer which content will not be modified must be `const`.
|
||||
This property is then controlled at compiler level.
|
||||
`const` variables are an important signal to readers that this variable isn't modified.
|
||||
Conversely, non-const variables are a signal to readers to watch out for modifications later on in the function.
|
||||
* If a function must be inlined, mention it explicitly,
|
||||
using project's own portable macros, such as `FORCE_INLINE_ATTR`,
|
||||
defined in `lib/common/compiler.h`.
|
||||
|
||||
### Debugging
|
||||
* **Assertions** are welcome, and should be used very liberally,
|
||||
to control any condition the code expects for its correct execution.
|
||||
These assertion checks will be run in debug builds, and disabled in production.
|
||||
* For traces, this project provides its own debug macros,
|
||||
in particular `DEBUGLOG(level, ...)`, defined in `lib/common/debug.h`.
|
||||
|
||||
### Code documentation
|
||||
* Avoid code documentation that merely repeats what the code is already stating.
|
||||
Whenever applicable, prefer employing the code as the primary way to convey explanations.
|
||||
Example 1 : `int nbTokens = n;` instead of `int i = n; /* i is a nb of tokens *./`.
|
||||
Example 2 : `assert(size > 0);` instead of `/* here, size should be positive */`.
|
||||
* At declaration level, the documentation explains how to use the function or variable
|
||||
and when applicable why it's needed, of the scenarios where it can be useful.
|
||||
* At implementation level, the documentation explains the general outline of the algorithm employed,
|
||||
and when applicable why this specific choice was preferred.
|
||||
|
||||
### General layout
|
||||
* 4 spaces for indentation rather than tabs
|
||||
* Code documentation shall directly precede function declaration or implementation
|
||||
* Function implementations and its code documentation should be preceded and followed by an empty line
|
||||
|
||||
|
||||
## License
|
||||
By contributing to Zstandard, you agree that your contributions will be licensed
|
||||
under both the [LICENSE](LICENSE) file and the [COPYING](COPYING) file in the root directory of this source tree.
|
||||
@@ -1,471 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
# verbose mode (print commands) on V=1 or VERBOSE=1
|
||||
Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
BUILDIR = build
|
||||
ZWRAPDIR = zlibWrapper
|
||||
TESTDIR = tests
|
||||
FUZZDIR = $(TESTDIR)/fuzz
|
||||
|
||||
# Define nul output
|
||||
VOID = /dev/null
|
||||
|
||||
# When cross-compiling from linux to windows, you might
|
||||
# need to specify this as "Windows." Fedora build fails
|
||||
# without it.
|
||||
#
|
||||
# Note: mingw-w64 build from linux to windows does not
|
||||
# fail on other tested distros (ubuntu, debian) even
|
||||
# without manually specifying the TARGET_SYSTEM.
|
||||
TARGET_SYSTEM ?= $(OS)
|
||||
CP ?= cp
|
||||
|
||||
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
|
||||
EXT =.exe
|
||||
else
|
||||
EXT =
|
||||
endif
|
||||
|
||||
## default: Build lib-release and zstd-release
|
||||
.PHONY: default
|
||||
default: lib-release zstd-release
|
||||
|
||||
.PHONY: all
|
||||
all: allmost examples manual contrib
|
||||
|
||||
.PHONY: allmost
|
||||
allmost: allzstd zlibwrapper
|
||||
|
||||
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||
.PHONY: allzstd
|
||||
allzstd: lib
|
||||
$(Q)$(MAKE) -C $(PRGDIR) all
|
||||
$(Q)$(MAKE) -C $(TESTDIR) all
|
||||
|
||||
.PHONY: all32
|
||||
all32:
|
||||
$(MAKE) -C $(PRGDIR) zstd32
|
||||
$(MAKE) -C $(TESTDIR) all32
|
||||
|
||||
.PHONY: lib lib-release lib-mt lib-nomt
|
||||
lib lib-release lib-mt lib-nomt:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
|
||||
.PHONY: zstd zstd-release
|
||||
zstd zstd-release:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT)
|
||||
|
||||
.PHONY: zstdmt
|
||||
zstdmt:
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
$(Q)$(CP) $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||
|
||||
.PHONY: zlibwrapper
|
||||
zlibwrapper: lib
|
||||
$(MAKE) -C $(ZWRAPDIR) all
|
||||
|
||||
## test: run long-duration tests
|
||||
.PHONY: test
|
||||
DEBUGLEVEL ?= 1
|
||||
test: MOREFLAGS += -g -Werror
|
||||
test:
|
||||
DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
|
||||
|
||||
## check: run basic tests for `zstd` cli
|
||||
.PHONY: check
|
||||
check:
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: automated_benchmarking
|
||||
automated_benchmarking:
|
||||
$(MAKE) -C $(TESTDIR) $@
|
||||
|
||||
.PHONY: benchmarking
|
||||
benchmarking: automated_benchmarking
|
||||
|
||||
## examples: build all examples in `examples/` directory
|
||||
.PHONY: examples
|
||||
examples: lib
|
||||
$(MAKE) -C examples all
|
||||
|
||||
## manual: generate API documentation in html format
|
||||
.PHONY: manual
|
||||
manual:
|
||||
$(MAKE) -C contrib/gen_html $@
|
||||
|
||||
## man: generate man page
|
||||
.PHONY: man
|
||||
man:
|
||||
$(MAKE) -C programs $@
|
||||
|
||||
## contrib: build all supported projects in `/contrib` directory
|
||||
.PHONY: contrib
|
||||
contrib: lib
|
||||
$(MAKE) -C contrib/pzstd all
|
||||
$(MAKE) -C contrib/seekable_format/examples all
|
||||
$(MAKE) -C contrib/seekable_format/tests test
|
||||
$(MAKE) -C contrib/largeNbDicts all
|
||||
$(MAKE) -C contrib/externalSequenceProducer all
|
||||
cd build/single_file_libs/ ; ./build_decoder_test.sh
|
||||
cd build/single_file_libs/ ; ./build_library_test.sh
|
||||
|
||||
.PHONY: cleanTabs
|
||||
cleanTabs:
|
||||
cd contrib; ./cleanTabs
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(TESTDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C examples/ $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/gen_html $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/pzstd $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/seekable_format/tests $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
|
||||
$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
|
||||
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
|
||||
$(Q)$(RM) -r lz4 cmakebuild mesonbuild install
|
||||
@echo Cleaning completed
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, macOS, Hurd and some BSD targets
|
||||
#------------------------------------------------------------------------------
|
||||
ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT% CYGWIN_NT% Haiku AIX,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
|
||||
|
||||
HOST_OS = POSIX
|
||||
|
||||
MKDIR ?= mkdir -p
|
||||
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($(HAVE_COLORNEVER), 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
#
|
||||
## list: Print all targets and their descriptions (if provided)
|
||||
.PHONY: list
|
||||
list:
|
||||
$(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
|
||||
| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
|
||||
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
|
||||
{ \
|
||||
printf "Target Name\tDescription\n"; \
|
||||
printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
|
||||
for target in $$TARGETS; do \
|
||||
line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
|
||||
description=$$(echo $$line | awk '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
|
||||
printf "$$target\t$$description\n"; \
|
||||
done \
|
||||
} | column -t -s $$'\t'
|
||||
|
||||
.PHONY: install armtest usan asan uasan msan asan32
|
||||
install:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
$(Q)$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(Q)$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
.PHONY: travis-install
|
||||
travis-install:
|
||||
$(MAKE) install PREFIX=~/install_test_dir
|
||||
|
||||
.PHONY: clangbuild-darwin-fat
|
||||
clangbuild-darwin-fat: clean
|
||||
clang -v
|
||||
CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release
|
||||
mv programs/zstd programs/zstd_arm64
|
||||
CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release
|
||||
mv programs/zstd programs/zstd_x64
|
||||
lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd
|
||||
|
||||
.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
|
||||
gcc5build: clean
|
||||
gcc-5 -v
|
||||
CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
|
||||
|
||||
gcc6build: clean
|
||||
gcc-6 -v
|
||||
CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
|
||||
|
||||
gcc7build: clean
|
||||
gcc-7 -v
|
||||
CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
|
||||
|
||||
clangbuild: clean
|
||||
clang -v
|
||||
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
|
||||
|
||||
m32build: clean
|
||||
gcc -v
|
||||
$(MAKE) all32
|
||||
|
||||
armbuild: clean
|
||||
CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allzstd
|
||||
|
||||
aarch64build: clean
|
||||
CC=aarch64-linux-gnu-gcc CFLAGS="-Werror -O0" $(MAKE) allzstd
|
||||
|
||||
ppcbuild: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m32 -Wno-attributes -Werror" $(MAKE) -j allzstd
|
||||
|
||||
ppc64build: clean
|
||||
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) -j allzstd
|
||||
|
||||
.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
|
||||
armfuzz: clean
|
||||
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
aarch64fuzz: clean
|
||||
ld -v
|
||||
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
ppcfuzz: clean
|
||||
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
ppc64fuzz: clean
|
||||
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
|
||||
cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
|
||||
cxxtest: clean
|
||||
$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)" # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
|
||||
|
||||
gcc5test: clean
|
||||
gcc-5 -v
|
||||
$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)"
|
||||
|
||||
gcc6test: clean
|
||||
gcc-6 -v
|
||||
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)"
|
||||
|
||||
armtest: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
|
||||
|
||||
aarch64test:
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
|
||||
|
||||
ppctest: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
|
||||
|
||||
ppc64test: clean
|
||||
$(MAKE) -C $(TESTDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
|
||||
|
||||
.PHONY: arm-ppc-compilation
|
||||
arm-ppc-compilation:
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)"
|
||||
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)"
|
||||
|
||||
regressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest
|
||||
|
||||
uasanregressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"
|
||||
|
||||
msanregressiontest:
|
||||
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory -Werror" CXXFLAGS="-O3 -fsanitize=memory -Werror"
|
||||
|
||||
update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
|
||||
update_regressionResults:
|
||||
$(MAKE) -j -C programs zstd
|
||||
$(MAKE) -j -C tests/regression test
|
||||
$(RM) -r $(REGRESS_RESULTS_DIR)
|
||||
$(MKDIR) $(REGRESS_RESULTS_DIR)
|
||||
./tests/regression/test \
|
||||
--cache tests/regression/cache \
|
||||
--output $(REGRESS_RESULTS_DIR)/results.csv \
|
||||
--zstd programs/zstd
|
||||
echo "Showing results differences"
|
||||
! diff tests/regression/results.csv $(REGRESS_RESULTS_DIR)/results.csv
|
||||
echo "Updating results.csv"
|
||||
$(CP) $(REGRESS_RESULTS_DIR)/results.csv tests/regression/results.csv
|
||||
|
||||
|
||||
# run UBsan with -fsanitize-recover=pointer-overflow
|
||||
# this only works with recent compilers such as gcc 8+
|
||||
usan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined -Werror $(MOREFLAGS)"
|
||||
|
||||
asan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
|
||||
|
||||
asan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
|
||||
|
||||
msan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
|
||||
|
||||
msan-%:
|
||||
$(MAKE) clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -j -C $(TESTDIR) HAVE_LZMA=0 $*
|
||||
|
||||
asan32: clean
|
||||
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
|
||||
|
||||
uasan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)"
|
||||
|
||||
uasan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
|
||||
|
||||
tsan-%: clean
|
||||
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
|
||||
|
||||
.PHONY: apt-install
|
||||
apt-install:
|
||||
# TODO: uncomment once issue 3011 is resolved and remove hack from Github Actions .yml
|
||||
# sudo apt-get update
|
||||
sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
|
||||
|
||||
.PHONY: apt-add-repo
|
||||
apt-add-repo:
|
||||
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
|
||||
sudo apt-get update -y -qq
|
||||
|
||||
.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
|
||||
ppcinstall:
|
||||
APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
|
||||
|
||||
arminstall:
|
||||
APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
|
||||
|
||||
valgrindinstall:
|
||||
APT_PACKAGES="valgrind" $(MAKE) apt-install
|
||||
|
||||
libc6install:
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc6install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc7install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
|
||||
|
||||
gcc8install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install
|
||||
|
||||
gpp6install: apt-add-repo
|
||||
APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
|
||||
|
||||
clang38install:
|
||||
APT_PACKAGES="clang-3.8" $(MAKE) apt-install
|
||||
|
||||
# Ubuntu 14.04 ships a too-old lz4
|
||||
lz4install:
|
||||
[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
|
||||
|
||||
endif
|
||||
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
|
||||
HOST_OS = MSYS
|
||||
endif
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
# target specific tests
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter MSYS POSIX,$(HOST_OS)))
|
||||
|
||||
CMAKE ?= cmake
|
||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname')))
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
|
||||
endif
|
||||
|
||||
.PHONY: cmakebuild
|
||||
cmakebuild:
|
||||
$(CMAKE) --version
|
||||
$(RM) -r cmakebuild install
|
||||
$(MKDIR) cmakebuild install
|
||||
cd cmakebuild; $(CMAKE) -Wdev -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS="-Werror -O0" -DCMAKE_INSTALL_PREFIX=install $(CMAKE_PARAMS) ../build/cmake
|
||||
$(CMAKE) --build cmakebuild --target install -- -j V=1
|
||||
cd cmakebuild; ctest -V -L Medium
|
||||
|
||||
MESON ?= meson
|
||||
NINJA ?= ninja
|
||||
|
||||
.PHONY: mesonbuild
|
||||
mesonbuild:
|
||||
$(MESON) setup \
|
||||
--buildtype=debugoptimized \
|
||||
-Db_lundef=false \
|
||||
-Dauto_features=enabled \
|
||||
-Dbin_programs=true \
|
||||
-Dbin_tests=true \
|
||||
-Dbin_contrib=true \
|
||||
-Ddefault_library=both \
|
||||
build/meson mesonbuild
|
||||
$(NINJA) -C mesonbuild/
|
||||
$(MESON) test -C mesonbuild/ --print-errorlogs
|
||||
$(MESON) install -C mesonbuild --destdir staging/
|
||||
|
||||
.PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
|
||||
c89build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c89 -Werror -Wno-attributes -Wpedantic -Wno-long-long -Wno-variadic-macros -O0" $(MAKE) lib zstd
|
||||
|
||||
gnu90build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=gnu90 -Werror -O0" $(MAKE) allmost
|
||||
|
||||
c99build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c99 -Werror -O0" $(MAKE) allmost
|
||||
|
||||
gnu99build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=gnu99 -Werror -O0" $(MAKE) allmost
|
||||
|
||||
c11build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-std=c11 -Werror -O0" $(MAKE) allmost
|
||||
|
||||
bmix64build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
bmix32build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
bmi32build: clean
|
||||
$(CC) -v
|
||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||
|
||||
# static analyzer test uses clang's scan-build
|
||||
# does not analyze zlibWrapper, due to detected issues in zlib source code
|
||||
staticAnalyze: SCANBUILD ?= scan-build
|
||||
staticAnalyze:
|
||||
$(CC) -v
|
||||
CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) zstd
|
||||
endif
|
||||
@@ -1,36 +0,0 @@
|
||||
// swift-tools-version:5.0
|
||||
// The swift-tools-version declares the minimum version of Swift required to build this package.
|
||||
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "zstd",
|
||||
platforms: [
|
||||
.macOS(.v10_10), .iOS(.v9), .tvOS(.v9)
|
||||
],
|
||||
products: [
|
||||
// Products define the executables and libraries a package produces, and make them visible to other packages.
|
||||
.library(
|
||||
name: "libzstd",
|
||||
targets: [ "libzstd" ])
|
||||
],
|
||||
dependencies: [
|
||||
// Dependencies declare other packages that this package depends on.
|
||||
// .package(url: /* package url */, from: "1.0.0"),
|
||||
],
|
||||
targets: [
|
||||
// Targets are the basic building blocks of a package. A target can define a module or a test suite.
|
||||
// Targets can depend on other targets in this package, and on products in packages this package depends on.
|
||||
.target(
|
||||
name: "libzstd",
|
||||
path: "lib",
|
||||
sources: [ "common", "compress", "decompress", "dictBuilder" ],
|
||||
publicHeadersPath: ".",
|
||||
cSettings: [
|
||||
.headerSearchPath(".")
|
||||
])
|
||||
],
|
||||
swiftLanguageVersions: [.v5],
|
||||
cLanguageStandard: .gnu11,
|
||||
cxxLanguageStandard: .gnucxx14
|
||||
)
|
||||
@@ -1,43 +0,0 @@
|
||||
Testing
|
||||
=======
|
||||
|
||||
Zstandard CI testing is split up into three sections:
|
||||
short, medium, and long tests.
|
||||
|
||||
Short Tests
|
||||
-----------
|
||||
Short tests run on CircleCI for new commits on every branch and pull request.
|
||||
They consist of the following tests:
|
||||
- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
|
||||
- Compilation on various versions of gcc, clang, and g++
|
||||
- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
|
||||
- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64
|
||||
|
||||
Medium Tests
|
||||
------------
|
||||
Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
|
||||
They consist of the following tests:
|
||||
- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
|
||||
Msan on x86_64
|
||||
- `tests/playTests.sh --test-large-data`
|
||||
- Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
|
||||
- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
|
||||
- Valgrind Test (`make -C tests test-valgrind`) (testing CLI and fuzzer under `valgrind`)
|
||||
- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64
|
||||
|
||||
Long Tests
|
||||
----------
|
||||
Long tests run on all commits to `release` branch,
|
||||
and once a day on the current version of `dev` branch,
|
||||
on TravisCI.
|
||||
They consist of the following tests:
|
||||
- Entire test suite (including fuzzers and some other specialized tests) on:
|
||||
- x86_64 and x86 with UBsan and Asan
|
||||
- x86_64 with Msan
|
||||
- ARM, AArch64, PowerPC, and PowerPC64
|
||||
- Streaming mode fuzzer with Tsan (for the `zstdmt` testing)
|
||||
- ZlibWrapper tests, including under valgrind
|
||||
- Versions test (ensuring `zstd` can decode files from all previous versions)
|
||||
- `pzstd` with asan and tsan, as well as in 32-bits mode
|
||||
- Testing `zstd` with legacy mode off
|
||||
- Entire test suite and make install on macOS
|
||||
@@ -1,26 +0,0 @@
|
||||
Zstandard Documentation
|
||||
=======================
|
||||
|
||||
This directory contains material defining the Zstandard format,
|
||||
as well as detailed instructions to use `zstd` library.
|
||||
|
||||
__`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format.
|
||||
Unfortunately, Github doesn't display `html` files in parsed format, just as source code.
|
||||
For a readable display of html API documentation of latest release,
|
||||
use this link: [https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html](https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html) .
|
||||
|
||||
__`zstd_compression_format.md`__ : This document defines the Zstandard compression format.
|
||||
Compliant decoders must adhere to this document,
|
||||
and compliant encoders must generate data that follows it.
|
||||
|
||||
Should you look for resources to develop your own port of Zstandard algorithm,
|
||||
you may find the following resources useful :
|
||||
|
||||
__`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
|
||||
compliant with the Zstandard compression format.
|
||||
It can be used, for example, to better understand the format,
|
||||
or as the basis for a separate implementation of Zstandard decoder.
|
||||
|
||||
[__`decode_corpus`__](https://github.com/facebook/zstd/tree/dev/tests#decodecorpus---tool-to-generate-zstandard-frames-for-decoder-testing) :
|
||||
This tool, stored in `/tests` directory, is able to generate random valid frames,
|
||||
which is useful if you wish to test your decoder and verify it fully supports the specification.
|
||||
@@ -1,148 +0,0 @@
|
||||
Decompressor Errata
|
||||
===================
|
||||
|
||||
This document captures known decompressor bugs, where the decompressor rejects a valid zstd frame.
|
||||
Each entry will contain:
|
||||
1. The last affected decompressor versions.
|
||||
2. The decompressor components affected.
|
||||
2. Whether the compressed frame could ever be produced by the reference compressor.
|
||||
3. An example frame (hexadecimal string when it can be short enough, link to golden file otherwise)
|
||||
4. A description of the bug.
|
||||
|
||||
The document is in reverse chronological order, with the bugs that affect the most recent zstd decompressor versions listed first.
|
||||
|
||||
|
||||
No sequence using the 2-bytes format
|
||||
------------------------------------------------
|
||||
|
||||
**Last affected version**: v1.5.5
|
||||
|
||||
**Affected decompressor component(s)**: Library & CLI
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
**Example Frame**: see zstd/tests/golden-decompression/zeroSeq_2B.zst
|
||||
|
||||
The zstd decoder incorrectly expects FSE tables when there are 0 sequences present in the block
|
||||
if the value 0 is encoded using the 2-bytes format.
|
||||
Instead, it should immediately end the sequence section, and move on to next block.
|
||||
|
||||
This situation was never generated by the reference compressor,
|
||||
because representing 0 sequences with the 2-bytes format is inefficient
|
||||
(the 1-byte format is always used in this case).
|
||||
|
||||
|
||||
Compressed block with a size of exactly 128 KB
|
||||
------------------------------------------------
|
||||
|
||||
**Last affected version**: v1.5.2
|
||||
|
||||
**Affected decompressor component(s)**: Library & CLI
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
**Example Frame**: see zstd/tests/golden-decompression/block-128k.zst
|
||||
|
||||
The zstd decoder incorrectly rejected blocks of type `Compressed_Block` when their size was exactly 128 KB.
|
||||
Note that `128 KB - 1` was accepted, and `128 KB + 1` is forbidden by the spec.
|
||||
|
||||
This type of block was never generated by the reference compressor.
|
||||
|
||||
These blocks used to be disallowed by the spec up until spec version 0.3.2 when the restriction was lifted by [PR#1689](https://github.com/facebook/zstd/pull/1689).
|
||||
|
||||
> A Compressed_Block has the extra restriction that Block_Size is always strictly less than the decompressed size. If this condition cannot be respected, the block must be sent uncompressed instead (Raw_Block).
|
||||
|
||||
|
||||
Compressed block with 0 literals and 0 sequences
|
||||
------------------------------------------------
|
||||
|
||||
**Last affected version**: v1.5.2
|
||||
|
||||
**Affected decompressor component(s)**: Library & CLI
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
**Example Frame**: `28b5 2ffd 2000 1500 0000 00`
|
||||
|
||||
The zstd decoder incorrectly rejected blocks of type `Compressed_Block` that encodes literals as `Raw_Literals_Block` with no literals, and has no sequences.
|
||||
|
||||
This type of block was never generated by the reference compressor.
|
||||
|
||||
Additionally, these blocks were disallowed by the spec up until spec version 0.3.2 when the restriction was lifted by [PR#1689](https://github.com/facebook/zstd/pull/1689).
|
||||
|
||||
> A Compressed_Block has the extra restriction that Block_Size is always strictly less than the decompressed size. If this condition cannot be respected, the block must be sent uncompressed instead (Raw_Block).
|
||||
|
||||
|
||||
First block is RLE block
|
||||
------------------------
|
||||
|
||||
**Last affected version**: v1.4.3
|
||||
|
||||
**Affected decompressor component(s)**: CLI only
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
**Example Frame**: `28b5 2ffd a001 0002 0002 0010 000b 0000 00`
|
||||
|
||||
The zstd CLI decompressor rejected cases where the first block was an RLE block whose `Block_Size` is 131072, and the frame contains more than one block.
|
||||
This only affected the zstd CLI, and not the library.
|
||||
|
||||
The example is an RLE block with 131072 bytes, followed by a second RLE block with 1 byte.
|
||||
|
||||
The compressor currently works around this limitation by explicitly avoiding producing RLE blocks as the first
|
||||
block.
|
||||
|
||||
https://github.com/facebook/zstd/blob/8814aa5bfa74f05a86e55e9d508da177a893ceeb/lib/compress/zstd_compress.c#L3527-L3535
|
||||
|
||||
|
||||
Tiny FSE Table & Block
|
||||
----------------------
|
||||
|
||||
**Last affected version**: v1.3.4
|
||||
|
||||
**Affected decompressor component(s)**: Library & CLI
|
||||
|
||||
**Produced by the reference compressor**: Possibly until version v1.3.4, but probably never
|
||||
|
||||
**Example Frame**: `28b5 2ffd 2027 c500 0080 f3f1 f0ec ebc6 c5c7 f09d 4300 0000 e0e0 0658 0100 603e 52`
|
||||
|
||||
The zstd library rejected blocks of type `Compressed_Block` whose offset of the last table with type `FSE_Compressed_Mode` was less than 4 bytes from the end of the block.
|
||||
|
||||
In more depth, let `Last_Table_Offset` be the offset in the compressed block (excluding the header) that
|
||||
the last table with type `FSE_Compressed_Mode` started. If `Block_Content - Last_Table_Offset < 4` then
|
||||
the buggy zstd decompressor would reject the block. This occurs when the last serialized table is 2 bytes
|
||||
and the bitstream size is 1 byte.
|
||||
|
||||
For example:
|
||||
* There is 1 sequence in the block
|
||||
* `Literals_Lengths_Mode` is `FSE_Compressed_Mode` & the serialized table size is 2 bytes
|
||||
* `Offsets_Mode` is `Predefined_Mode`
|
||||
* `Match_Lengths_Mode` is `Predefined_Mode`
|
||||
* The bitstream is 1 byte. E.g. there is only one sequence and it fits in 1 byte.
|
||||
|
||||
The total `Block_Content` is `5` bytes, and `Last_Table_Offset` is `2`.
|
||||
|
||||
See the compressor workaround code:
|
||||
|
||||
https://github.com/facebook/zstd/blob/8814aa5bfa74f05a86e55e9d508da177a893ceeb/lib/compress/zstd_compress.c#L2667-L2682
|
||||
|
||||
Magicless format
|
||||
----------------------
|
||||
|
||||
**Last affected version**: v1.5.5
|
||||
|
||||
**Affected decompressor component(s)**: Library
|
||||
|
||||
**Produced by the reference compressor**: Yes (example: https://gist.github.com/embg/9940726094f4cf2cef162cffe9319232)
|
||||
|
||||
**Example Frame**: `27 b5 2f fd 00 03 19 00 00 66 6f 6f 3f ba c4 59`
|
||||
|
||||
v1.5.6 fixes several bugs in which the magicless-format decoder rejects valid frames.
|
||||
These include but are not limited to:
|
||||
* Valid frames that happen to begin with a legacy magic number (little-endian)
|
||||
* Valid frames that happen to begin with a skippable magic number (little-endian)
|
||||
|
||||
If you are affected by this issue and cannot update to v1.5.6 or later, there is a
|
||||
workaround to recover affected data. Simply prepend the ZSTD magic number
|
||||
`0xFD2FB528` (little-endian) to your data and decompress using the standard-format
|
||||
decoder.
|
||||
@@ -1,80 +0,0 @@
|
||||
Decompressor Permissiveness to Invalid Data
|
||||
===========================================
|
||||
|
||||
This document describes the behavior of the reference decompressor in cases
|
||||
where it accepts formally invalid data instead of reporting an error.
|
||||
|
||||
While the reference decompressor *must* decode any compliant frame following
|
||||
the specification, its ability to detect erroneous data is on a best effort
|
||||
basis: the decoder may accept input data that would be formally invalid,
|
||||
when it causes no risk to the decoder, and which detection would cost too much
|
||||
complexity or speed regression.
|
||||
|
||||
In practice, the vast majority of invalid data are detected, if only because
|
||||
many corruption events are dangerous for the decoder process (such as
|
||||
requesting an out-of-bound memory access) and many more are easy to check.
|
||||
|
||||
This document lists a few known cases where invalid data was formerly accepted
|
||||
by the decoder, and what has changed since.
|
||||
|
||||
|
||||
Truncated Huffman states
|
||||
------------------------
|
||||
|
||||
**Last affected version**: v1.5.6
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
**Example Frame**: `28b5 2ffd 0000 5500 0072 8001 0420 7e1f 02aa 00`
|
||||
|
||||
When using FSE-compressed Huffman weights, the compressed weight bitstream
|
||||
could contain fewer bits than necessary to decode the initial states.
|
||||
|
||||
The reference decompressor up to v1.5.6 will decode truncated or missing
|
||||
initial states as zero, which can result in a valid Huffman tree if only
|
||||
the second state is truncated.
|
||||
|
||||
In newer versions, truncated initial states are reported as a corruption
|
||||
error by the decoder.
|
||||
|
||||
|
||||
Offset == 0
|
||||
-----------
|
||||
|
||||
**Last affected version**: v1.5.5
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
**Example Frame**: `28b5 2ffd 0000 4500 0008 0002 002f 430b ae`
|
||||
|
||||
If a sequence is decoded with `literals_length = 0` and `offset_value = 3`
|
||||
while `Repeated_Offset_1 = 1`, the computed offset will be `0`, which is
|
||||
invalid.
|
||||
|
||||
The reference decompressor up to v1.5.5 processes this case as if the computed
|
||||
offset was `1`, including inserting `1` into the repeated offset list.
|
||||
This prevents the output buffer from remaining uninitialized, thus denying a
|
||||
potential attack vector from an untrusted source.
|
||||
However, in the rare case where this scenario would be the outcome of a
|
||||
transmission or storage error, the decoder relies on the checksum to detect
|
||||
the error.
|
||||
|
||||
In newer versions, this case is always detected and reported as a corruption error.
|
||||
|
||||
|
||||
Non-zeroes reserved bits
|
||||
------------------------
|
||||
|
||||
**Last affected version**: v1.5.5
|
||||
|
||||
**Produced by the reference compressor**: No
|
||||
|
||||
The Sequences section of each block has a header, and one of its elements is a
|
||||
byte, which describes the compression mode of each symbol.
|
||||
This byte contains 2 reserved bits which must be set to zero.
|
||||
|
||||
The reference decompressor up to v1.5.5 just ignores these 2 bits.
|
||||
This behavior has no consequence for the rest of the frame decoding process.
|
||||
|
||||
In newer versions, the 2 reserved bits are actively checked for value zero,
|
||||
and the decoder reports a corruption error if they are not.
|
||||
@@ -1,62 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
ZSTD ?= zstd # note: requires zstd installation on local system
|
||||
|
||||
UNAME?= $(shell sh -c 'MSYSTEM="MSYS" uname')
|
||||
ifeq ($(UNAME), SunOS)
|
||||
DIFF ?= gdiff
|
||||
else
|
||||
DIFF ?= diff
|
||||
endif
|
||||
|
||||
HARNESS_FILES=*.c
|
||||
|
||||
MULTITHREAD_LDFLAGS = -pthread
|
||||
DEBUGFLAGS= -g -DZSTD_DEBUG=1
|
||||
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||
CFLAGS ?= -O2
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum \
|
||||
-Wredundant-decls -Wstrict-prototypes -Wundef \
|
||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||
-std=c99
|
||||
CFLAGS += $(DEBUGFLAGS)
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
|
||||
|
||||
harness: $(HARNESS_FILES)
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
clean:
|
||||
@$(RM) harness *.o
|
||||
@$(RM) -rf harness.dSYM # MacOS specific
|
||||
|
||||
test: harness
|
||||
#
|
||||
# Testing single-file decompression with educational decoder
|
||||
#
|
||||
@$(ZSTD) -f README.md -o tmp.zst
|
||||
@./harness tmp.zst tmp
|
||||
@$(DIFF) -s tmp README.md
|
||||
@$(RM) tmp*
|
||||
#
|
||||
# Testing dictionary decompression with education decoder
|
||||
#
|
||||
# note : files are presented multiple for training, to reach minimum threshold
|
||||
@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
|
||||
harness.c zstd_decompress.c zstd_decompress.h README.md \
|
||||
harness.c zstd_decompress.c zstd_decompress.h README.md \
|
||||
-o dictionary
|
||||
@$(ZSTD) -f README.md -D dictionary -o tmp.zst
|
||||
@./harness tmp.zst tmp dictionary
|
||||
@$(DIFF) -s tmp README.md
|
||||
@$(RM) tmp* dictionary
|
||||
@@ -1,36 +0,0 @@
|
||||
Educational Decoder
|
||||
===================
|
||||
|
||||
`zstd_decompress.c` is a self-contained implementation in C99 of a decoder,
|
||||
according to the [Zstandard format specification].
|
||||
While it does not implement as many features as the reference decoder,
|
||||
such as the streaming API or content checksums, it is written to be easy to
|
||||
follow and understand, to help understand how the Zstandard format works.
|
||||
It's laid out to match the [format specification],
|
||||
so it can be used to understand how complex segments could be implemented.
|
||||
It also contains implementations of Huffman and FSE table decoding.
|
||||
|
||||
[Zstandard format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
|
||||
[format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
|
||||
|
||||
While the library's primary objective is code clarity,
|
||||
it also happens to compile into a small object file.
|
||||
The object file can be made even smaller by removing error messages,
|
||||
using the macro directive `ZDEC_NO_MESSAGE` at compilation time.
|
||||
This can be reduced even further by foregoing dictionary support,
|
||||
by defining `ZDEC_NO_DICTIONARY`.
|
||||
|
||||
`harness.c` provides a simple test harness around the decoder:
|
||||
|
||||
harness <input-file> <output-file> [dictionary]
|
||||
|
||||
As an additional resource to be used with this decoder,
|
||||
see the `decodecorpus` tool in the [tests] directory.
|
||||
It generates valid Zstandard frames that can be used to verify
|
||||
a Zstandard decoder implementation.
|
||||
Note that to use the tool to verify this decoder implementation,
|
||||
the --content-size flag should be set,
|
||||
as this decoder does not handle streaming decoding,
|
||||
and so it must know the decompressed size in advance.
|
||||
|
||||
[tests]: https://github.com/facebook/zstd/blob/dev/tests/
|
||||
@@ -1,119 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "zstd_decompress.h"
|
||||
|
||||
typedef unsigned char u8;
|
||||
|
||||
// If the data doesn't have decompressed size with it, fallback on assuming the
|
||||
// compression ratio is at most 16
|
||||
#define MAX_COMPRESSION_RATIO (16)
|
||||
|
||||
// Protect against allocating too much memory for output
|
||||
#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
|
||||
|
||||
// Error message then exit
|
||||
#define ERR_OUT(...) { fprintf(stderr, __VA_ARGS__); exit(1); }
|
||||
|
||||
|
||||
typedef struct {
|
||||
u8* address;
|
||||
size_t size;
|
||||
} buffer_s;
|
||||
|
||||
static void freeBuffer(buffer_s b) { free(b.address); }
|
||||
|
||||
static buffer_s read_file(const char *path)
|
||||
{
|
||||
FILE* const f = fopen(path, "rb");
|
||||
if (!f) ERR_OUT("failed to open file %s \n", path);
|
||||
|
||||
fseek(f, 0L, SEEK_END);
|
||||
size_t const size = (size_t)ftell(f);
|
||||
rewind(f);
|
||||
|
||||
void* const ptr = malloc(size);
|
||||
if (!ptr) ERR_OUT("failed to allocate memory to hold %s \n", path);
|
||||
|
||||
size_t const read = fread(ptr, 1, size, f);
|
||||
if (read != size) ERR_OUT("error while reading file %s \n", path);
|
||||
|
||||
fclose(f);
|
||||
buffer_s const b = { ptr, size };
|
||||
return b;
|
||||
}
|
||||
|
||||
static void write_file(const char* path, const u8* ptr, size_t size)
|
||||
{
|
||||
FILE* const f = fopen(path, "wb");
|
||||
if (!f) ERR_OUT("failed to open file %s \n", path);
|
||||
|
||||
size_t written = 0;
|
||||
while (written < size) {
|
||||
written += fwrite(ptr+written, 1, size, f);
|
||||
if (ferror(f)) ERR_OUT("error while writing file %s\n", path);
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc < 3)
|
||||
ERR_OUT("usage: %s <file.zst> <out_path> [dictionary] \n", argv[0]);
|
||||
|
||||
buffer_s const input = read_file(argv[1]);
|
||||
|
||||
buffer_s dict = { NULL, 0 };
|
||||
if (argc >= 4) {
|
||||
dict = read_file(argv[3]);
|
||||
}
|
||||
|
||||
size_t out_capacity = ZSTD_get_decompressed_size(input.address, input.size);
|
||||
if (out_capacity == (size_t)-1) {
|
||||
out_capacity = MAX_COMPRESSION_RATIO * input.size;
|
||||
fprintf(stderr, "WARNING: Compressed data does not contain "
|
||||
"decompressed size, going to assume the compression "
|
||||
"ratio is at most %d (decompressed size of at most "
|
||||
"%u) \n",
|
||||
MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
|
||||
}
|
||||
if (out_capacity > MAX_OUTPUT_SIZE)
|
||||
ERR_OUT("Required output size too large for this implementation \n");
|
||||
|
||||
u8* const output = malloc(out_capacity);
|
||||
if (!output) ERR_OUT("failed to allocate memory \n");
|
||||
|
||||
dictionary_t* const parsed_dict = create_dictionary();
|
||||
if (dict.size) {
|
||||
#if defined (ZDEC_NO_DICTIONARY)
|
||||
printf("dict.size = %zu \n", dict.size);
|
||||
ERR_OUT("no dictionary support \n");
|
||||
#else
|
||||
parse_dictionary(parsed_dict, dict.address, dict.size);
|
||||
#endif
|
||||
}
|
||||
size_t const decompressed_size =
|
||||
ZSTD_decompress_with_dict(output, out_capacity,
|
||||
input.address, input.size,
|
||||
parsed_dict);
|
||||
|
||||
free_dictionary(parsed_dict);
|
||||
|
||||
write_file(argv[2], output, decompressed_size);
|
||||
|
||||
freeBuffer(input);
|
||||
freeBuffer(dict);
|
||||
free(output);
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,61 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
/******* EXPOSED TYPES ********************************************************/
|
||||
/*
|
||||
* Contains the parsed contents of a dictionary
|
||||
* This includes Huffman and FSE tables used for decoding and data on offsets
|
||||
*/
|
||||
typedef struct dictionary_s dictionary_t;
|
||||
/******* END EXPOSED TYPES ****************************************************/
|
||||
|
||||
/******* DECOMPRESSION FUNCTIONS **********************************************/
|
||||
/// Zstandard decompression functions.
|
||||
/// `dst` must point to a space at least as large as the reconstructed output.
|
||||
size_t ZSTD_decompress(void *const dst, const size_t dst_len,
|
||||
const void *const src, const size_t src_len);
|
||||
|
||||
/// If `dict != NULL` and `dict_len >= 8`, does the same thing as
|
||||
/// `ZSTD_decompress` but uses the provided dict
|
||||
size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
|
||||
const void *const src, const size_t src_len,
|
||||
dictionary_t* parsed_dict);
|
||||
|
||||
/// Get the decompressed size of an input stream so memory can be allocated in
|
||||
/// advance
|
||||
/// Returns -1 if the size can't be determined
|
||||
/// Assumes decompression of a single frame
|
||||
size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
|
||||
/******* END DECOMPRESSION FUNCTIONS ******************************************/
|
||||
|
||||
/******* DICTIONARY MANAGEMENT ***********************************************/
|
||||
/*
|
||||
* Return a valid dictionary_t pointer for use with dictionary initialization
|
||||
* or decompression
|
||||
*/
|
||||
dictionary_t* create_dictionary(void);
|
||||
|
||||
/*
|
||||
* Parse a provided dictionary blob for use in decompression
|
||||
* `src` -- must point to memory space representing the dictionary
|
||||
* `src_len` -- must provide the dictionary size
|
||||
* `dict` -- will contain the parsed contents of the dictionary and
|
||||
* can be used for decompression
|
||||
*/
|
||||
void parse_dictionary(dictionary_t *const dict, const void *src,
|
||||
size_t src_len);
|
||||
|
||||
/*
|
||||
* Free internal Huffman tables, FSE tables, and dictionary content
|
||||
*/
|
||||
void free_dictionary(dictionary_t *const dict);
|
||||
/******* END DICTIONARY MANAGEMENT *******************************************/
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 72 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 68 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 26 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 33 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 88 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 89 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 96 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 92 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,93 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
LIBDIR =../lib
|
||||
CPPFLAGS += -I$(LIBDIR)
|
||||
LIB = $(LIBDIR)/libzstd.a
|
||||
|
||||
|
||||
.PHONY: default
|
||||
default: all
|
||||
|
||||
.PHONY: all
|
||||
all: simple_compression simple_decompression \
|
||||
multiple_simple_compression\
|
||||
dictionary_compression dictionary_decompression \
|
||||
streaming_compression streaming_decompression \
|
||||
multiple_streaming_compression streaming_memory_usage
|
||||
|
||||
$(LIB) :
|
||||
$(MAKE) -C $(LIBDIR) libzstd.a
|
||||
|
||||
simple_compression.o: common.h
|
||||
simple_compression : $(LIB)
|
||||
|
||||
simple_decompression.o: common.h
|
||||
simple_decompression : $(LIB)
|
||||
|
||||
multiple_simple_compression.o: common.h
|
||||
multiple_simple_compression : $(LIB)
|
||||
|
||||
dictionary_compression.o: common.h
|
||||
dictionary_compression : $(LIB)
|
||||
|
||||
dictionary_decompression.o: common.h
|
||||
dictionary_decompression : $(LIB)
|
||||
|
||||
streaming_compression.o: common.h
|
||||
streaming_compression : $(LIB)
|
||||
|
||||
multiple_streaming_compression.o: common.h
|
||||
multiple_streaming_compression : $(LIB)
|
||||
|
||||
streaming_decompression.o: common.h
|
||||
streaming_decompression : $(LIB)
|
||||
|
||||
streaming_memory_usage.o: common.h
|
||||
streaming_memory_usage : $(LIB)
|
||||
|
||||
|
||||
.PHONY:clean
|
||||
clean:
|
||||
@$(RM) core *.o tmp* result* *.zst \
|
||||
simple_compression simple_decompression \
|
||||
multiple_simple_compression \
|
||||
dictionary_compression dictionary_decompression \
|
||||
streaming_compression streaming_decompression \
|
||||
multiple_streaming_compression streaming_memory_usage
|
||||
@echo Cleaning completed
|
||||
|
||||
.PHONY:test
|
||||
test: all
|
||||
cp README.md tmp
|
||||
cp Makefile tmp2
|
||||
@echo -- Simple compression tests
|
||||
./simple_compression tmp
|
||||
./simple_decompression tmp.zst
|
||||
./multiple_simple_compression *.c
|
||||
./streaming_decompression tmp.zst > /dev/null
|
||||
@echo -- Streaming memory usage
|
||||
./streaming_memory_usage
|
||||
@echo -- Streaming compression tests
|
||||
./streaming_compression tmp
|
||||
./streaming_decompression tmp.zst > /dev/null
|
||||
@echo -- Edge cases detection
|
||||
! ./streaming_decompression tmp # invalid input, must fail
|
||||
! ./simple_decompression tmp # invalid input, must fail
|
||||
touch tmpNull # create 0-size file
|
||||
./simple_compression tmpNull
|
||||
./simple_decompression tmpNull.zst # 0-size frame : must work
|
||||
@echo -- Multiple streaming tests
|
||||
./multiple_streaming_compression *.c
|
||||
@echo -- Dictionary compression tests
|
||||
./dictionary_compression tmp2 tmp README.md
|
||||
./dictionary_decompression tmp2.zst tmp.zst README.md
|
||||
$(RM) tmp* *.zst
|
||||
@echo tests completed
|
||||
@@ -1,46 +0,0 @@
|
||||
Zstandard library : usage examples
|
||||
==================================
|
||||
|
||||
- [Simple compression](simple_compression.c) :
|
||||
Compress a single file.
|
||||
Introduces usage of : `ZSTD_compress()`
|
||||
|
||||
- [Simple decompression](simple_decompression.c) :
|
||||
Decompress a single file.
|
||||
Only compatible with simple compression.
|
||||
Result remains in memory.
|
||||
Introduces usage of : `ZSTD_decompress()`
|
||||
|
||||
- [Multiple simple compression](multiple_simple_compression.c) :
|
||||
Compress multiple files (in simple mode) in a single command line.
|
||||
Demonstrates memory preservation technique that
|
||||
minimizes malloc()/free() calls by re-using existing resources.
|
||||
Introduces usage of : `ZSTD_compressCCtx()`
|
||||
|
||||
- [Streaming memory usage](streaming_memory_usage.c) :
|
||||
Provides amount of memory used by streaming context.
|
||||
Introduces usage of : `ZSTD_sizeof_CStream()`
|
||||
|
||||
- [Streaming compression](streaming_compression.c) :
|
||||
Compress a single file.
|
||||
Introduces usage of : `ZSTD_compressStream()`
|
||||
|
||||
- [Multiple Streaming compression](multiple_streaming_compression.c) :
|
||||
Compress multiple files (in streaming mode) in a single command line.
|
||||
Introduces memory usage preservation technique,
|
||||
reducing impact of malloc()/free() and memset() by re-using existing resources.
|
||||
|
||||
- [Streaming decompression](streaming_decompression.c) :
|
||||
Decompress a single file compressed by zstd.
|
||||
Compatible with both simple and streaming compression.
|
||||
Result is sent to stdout.
|
||||
Introduces usage of : `ZSTD_decompressStream()`
|
||||
|
||||
- [Dictionary compression](dictionary_compression.c) :
|
||||
Compress multiple files using the same dictionary.
|
||||
Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()`
|
||||
|
||||
- [Dictionary decompression](dictionary_decompression.c) :
|
||||
Decompress multiple files using the same dictionary.
|
||||
Result remains in memory.
|
||||
Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()`
|
||||
@@ -1,246 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This header file has common utility functions used in examples.
|
||||
*/
|
||||
#ifndef COMMON_H
|
||||
#define COMMON_H
|
||||
|
||||
#include <stdlib.h> // malloc, free, exit
|
||||
#include <stdio.h> // fprintf, perror, fopen, etc.
|
||||
#include <string.h> // strerror
|
||||
#include <errno.h> // errno
|
||||
#include <sys/stat.h> // stat
|
||||
#include <zstd.h>
|
||||
|
||||
|
||||
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
|
||||
#if defined(__GNUC__)
|
||||
# define UNUSED_ATTR __attribute__((unused))
|
||||
#else
|
||||
# define UNUSED_ATTR
|
||||
#endif
|
||||
|
||||
#define HEADER_FUNCTION static UNUSED_ATTR
|
||||
|
||||
|
||||
/*
|
||||
* Define the returned error code from utility functions.
|
||||
*/
|
||||
typedef enum {
|
||||
ERROR_fsize = 1,
|
||||
ERROR_fopen = 2,
|
||||
ERROR_fclose = 3,
|
||||
ERROR_fread = 4,
|
||||
ERROR_fwrite = 5,
|
||||
ERROR_loadFile = 6,
|
||||
ERROR_saveFile = 7,
|
||||
ERROR_malloc = 8,
|
||||
ERROR_largeFile = 9,
|
||||
} COMMON_ErrorCode;
|
||||
|
||||
/*! CHECK
|
||||
* Check that the condition holds. If it doesn't print a message and die.
|
||||
*/
|
||||
#define CHECK(cond, ...) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
fprintf(stderr, \
|
||||
"%s:%d CHECK(%s) failed: ", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
#cond); \
|
||||
fprintf(stderr, "" __VA_ARGS__); \
|
||||
fprintf(stderr, "\n"); \
|
||||
exit(1); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/*! CHECK_ZSTD
|
||||
* Check the zstd error code and die if an error occurred after printing a
|
||||
* message.
|
||||
*/
|
||||
#define CHECK_ZSTD(fn) \
|
||||
do { \
|
||||
size_t const err = (fn); \
|
||||
CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
|
||||
} while (0)
|
||||
|
||||
/*! fsize_orDie() :
|
||||
* Get the size of a given file path.
|
||||
*
|
||||
* @return The size of a given file path.
|
||||
*/
|
||||
HEADER_FUNCTION size_t fsize_orDie(const char *filename)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(filename, &st) != 0) {
|
||||
/* error */
|
||||
perror(filename);
|
||||
exit(ERROR_fsize);
|
||||
}
|
||||
|
||||
off_t const fileSize = st.st_size;
|
||||
size_t const size = (size_t)fileSize;
|
||||
/* 1. fileSize should be non-negative,
|
||||
* 2. if off_t -> size_t type conversion results in discrepancy,
|
||||
* the file size is too large for type size_t.
|
||||
*/
|
||||
if ((fileSize < 0) || (fileSize != (off_t)size)) {
|
||||
fprintf(stderr, "%s : filesize too large \n", filename);
|
||||
exit(ERROR_largeFile);
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
/*! fopen_orDie() :
|
||||
* Open a file using given file path and open option.
|
||||
*
|
||||
* @return If successful this function will return a FILE pointer to an
|
||||
* opened file otherwise it sends an error to stderr and exits.
|
||||
*/
|
||||
HEADER_FUNCTION FILE* fopen_orDie(const char *filename, const char *instruction)
|
||||
{
|
||||
FILE* const inFile = fopen(filename, instruction);
|
||||
if (inFile) return inFile;
|
||||
/* error */
|
||||
perror(filename);
|
||||
exit(ERROR_fopen);
|
||||
}
|
||||
|
||||
/*! fclose_orDie() :
|
||||
* Close an opened file using given FILE pointer.
|
||||
*/
|
||||
HEADER_FUNCTION void fclose_orDie(FILE* file)
|
||||
{
|
||||
if (!fclose(file)) { return; };
|
||||
/* error */
|
||||
perror("fclose");
|
||||
exit(ERROR_fclose);
|
||||
}
|
||||
|
||||
/*! fread_orDie() :
|
||||
*
|
||||
* Read sizeToRead bytes from a given file, storing them at the
|
||||
* location given by buffer.
|
||||
*
|
||||
* @return The number of bytes read.
|
||||
*/
|
||||
HEADER_FUNCTION size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
|
||||
{
|
||||
size_t const readSize = fread(buffer, 1, sizeToRead, file);
|
||||
if (readSize == sizeToRead) return readSize; /* good */
|
||||
if (feof(file)) return readSize; /* good, reached end of file */
|
||||
/* error */
|
||||
perror("fread");
|
||||
exit(ERROR_fread);
|
||||
}
|
||||
|
||||
/*! fwrite_orDie() :
|
||||
*
|
||||
* Write sizeToWrite bytes to a file pointed to by file, obtaining
|
||||
* them from a location given by buffer.
|
||||
*
|
||||
* Note: This function will send an error to stderr and exit if it
|
||||
* cannot write data to the given file pointer.
|
||||
*
|
||||
* @return The number of bytes written.
|
||||
*/
|
||||
HEADER_FUNCTION size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
|
||||
{
|
||||
size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
|
||||
if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
|
||||
/* error */
|
||||
perror("fwrite");
|
||||
exit(ERROR_fwrite);
|
||||
}
|
||||
|
||||
/*! malloc_orDie() :
|
||||
* Allocate memory.
|
||||
*
|
||||
* @return If successful this function returns a pointer to allo-
|
||||
* cated memory. If there is an error, this function will send that
|
||||
* error to stderr and exit.
|
||||
*/
|
||||
HEADER_FUNCTION void* malloc_orDie(size_t size)
|
||||
{
|
||||
void* const buff = malloc(size);
|
||||
if (buff) return buff;
|
||||
/* error */
|
||||
perror("malloc");
|
||||
exit(ERROR_malloc);
|
||||
}
|
||||
|
||||
/*! loadFile_orDie() :
|
||||
* load file into buffer (memory).
|
||||
*
|
||||
* Note: This function will send an error to stderr and exit if it
|
||||
* cannot read data from the given file path.
|
||||
*
|
||||
* @return If successful this function will load file into buffer and
|
||||
* return file size, otherwise it will printout an error to stderr and exit.
|
||||
*/
|
||||
HEADER_FUNCTION size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
|
||||
{
|
||||
size_t const fileSize = fsize_orDie(fileName);
|
||||
CHECK(fileSize <= bufferSize, "File too large!");
|
||||
|
||||
FILE* const inFile = fopen_orDie(fileName, "rb");
|
||||
size_t const readSize = fread(buffer, 1, fileSize, inFile);
|
||||
if (readSize != (size_t)fileSize) {
|
||||
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
|
||||
exit(ERROR_fread);
|
||||
}
|
||||
fclose(inFile); /* can't fail, read only */
|
||||
return fileSize;
|
||||
}
|
||||
|
||||
/*! mallocAndLoadFile_orDie() :
|
||||
* allocate memory buffer and then load file into it.
|
||||
*
|
||||
* Note: This function will send an error to stderr and exit if memory allocation
|
||||
* fails or it cannot read data from the given file path.
|
||||
*
|
||||
* @return If successful this function will return buffer and bufferSize(=fileSize),
|
||||
* otherwise it will printout an error to stderr and exit.
|
||||
*/
|
||||
HEADER_FUNCTION void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize)
|
||||
{
|
||||
size_t const fileSize = fsize_orDie(fileName);
|
||||
*bufferSize = fileSize;
|
||||
void* const buffer = malloc_orDie(*bufferSize);
|
||||
loadFile_orDie(fileName, buffer, *bufferSize);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/*! saveFile_orDie() :
|
||||
*
|
||||
* Save buffSize bytes to a given file path, obtaining them from a location pointed
|
||||
* to by buff.
|
||||
*
|
||||
* Note: This function will send an error to stderr and exit if it
|
||||
* cannot write to a given file.
|
||||
*/
|
||||
HEADER_FUNCTION void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
|
||||
{
|
||||
FILE* const oFile = fopen_orDie(fileName, "wb");
|
||||
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
|
||||
if (wSize != (size_t)buffSize) {
|
||||
fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
|
||||
exit(ERROR_fwrite);
|
||||
}
|
||||
if (fclose(oFile)) {
|
||||
perror(fileName);
|
||||
exit(ERROR_fclose);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,107 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
**/
|
||||
|
||||
/* This example deals with Dictionary compression,
|
||||
* its counterpart is `examples/dictionary_decompression.c` .
|
||||
* These examples presume that a dictionary already exists.
|
||||
* The main method to create a dictionary is `zstd --train`,
|
||||
* look at the CLI documentation for details.
|
||||
* Another possible method is to employ dictionary training API,
|
||||
* published in `lib/zdict.h` .
|
||||
**/
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <string.h> // memset, strcat
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
/* createDict() :
|
||||
** `dictFileName` is supposed already created using `zstd --train` */
|
||||
static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
|
||||
{
|
||||
size_t dictSize;
|
||||
printf("loading dictionary %s \n", dictFileName);
|
||||
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
|
||||
CHECK(cdict != NULL, "ZSTD_createCDict() failed!");
|
||||
free(dictBuffer);
|
||||
return cdict;
|
||||
}
|
||||
|
||||
|
||||
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
|
||||
{
|
||||
size_t fSize;
|
||||
void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
|
||||
size_t const cBuffSize = ZSTD_compressBound(fSize);
|
||||
void* const cBuff = malloc_orDie(cBuffSize);
|
||||
|
||||
/* Compress using the dictionary.
|
||||
* This function writes the dictionary id, and content size into the header.
|
||||
* But, it doesn't use a checksum. You can control these options using the
|
||||
* advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(),
|
||||
* and ZSTD_compress2().
|
||||
*/
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
|
||||
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
|
||||
CHECK_ZSTD(cSize);
|
||||
|
||||
saveFile_orDie(oname, cBuff, cSize);
|
||||
|
||||
/* success */
|
||||
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
|
||||
|
||||
ZSTD_freeCCtx(cctx); /* never fails */
|
||||
free(fBuff);
|
||||
free(cBuff);
|
||||
}
|
||||
|
||||
|
||||
static char* createOutFilename_orDie(const char* filename)
|
||||
{
|
||||
size_t const inL = strlen(filename);
|
||||
size_t const outL = inL + 5;
|
||||
void* outSpace = malloc_orDie(outL);
|
||||
memset(outSpace, 0, outL);
|
||||
strcat(outSpace, filename);
|
||||
strcat(outSpace, ".zst");
|
||||
return (char*)outSpace;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
int const cLevel = 3;
|
||||
|
||||
if (argc<3) {
|
||||
fprintf(stderr, "wrong arguments\n");
|
||||
fprintf(stderr, "usage:\n");
|
||||
fprintf(stderr, "%s [FILES] dictionary\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* load dictionary only once */
|
||||
const char* const dictName = argv[argc-1];
|
||||
ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel);
|
||||
|
||||
int u;
|
||||
for (u=1; u<argc-1; u++) {
|
||||
const char* inFilename = argv[u];
|
||||
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||
compress(inFilename, outFilename, dictPtr);
|
||||
free(outFilename);
|
||||
}
|
||||
|
||||
ZSTD_freeCDict(dictPtr);
|
||||
printf("All %u files compressed. \n", argc-2);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
/* createDict() :
|
||||
`dictFileName` is supposed to have been created using `zstd --train` */
|
||||
static ZSTD_DDict* createDict_orDie(const char* dictFileName)
|
||||
{
|
||||
size_t dictSize;
|
||||
printf("loading dictionary %s \n", dictFileName);
|
||||
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
|
||||
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
|
||||
CHECK(ddict != NULL, "ZSTD_createDDict() failed!");
|
||||
free(dictBuffer);
|
||||
return ddict;
|
||||
}
|
||||
|
||||
static void decompress(const char* fname, const ZSTD_DDict* ddict)
|
||||
{
|
||||
size_t cSize;
|
||||
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
|
||||
/* Read the content size from the frame header. For simplicity we require
|
||||
* that it is always present. By default, zstd will write the content size
|
||||
* in the header when it is known. If you can't guarantee that the frame
|
||||
* content size is always written into the header, either use streaming
|
||||
* decompression, or ZSTD_decompressBound().
|
||||
*/
|
||||
unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
|
||||
CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
|
||||
CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
|
||||
void* const rBuff = malloc_orDie((size_t)rSize);
|
||||
|
||||
/* Check that the dictionary ID matches.
|
||||
* If a non-zstd dictionary is used, then both will be zero.
|
||||
* By default zstd always writes the dictionary ID into the frame.
|
||||
* Zstd will check if there is a dictionary ID mismatch as well.
|
||||
*/
|
||||
unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict);
|
||||
unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize);
|
||||
CHECK(actualDictID == expectedDictID,
|
||||
"DictID mismatch: expected %u got %u",
|
||||
expectedDictID,
|
||||
actualDictID);
|
||||
|
||||
/* Decompress using the dictionary.
|
||||
* If you need to control the decompression parameters, then use the
|
||||
* advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and
|
||||
* ZSTD_decompressDCtx().
|
||||
*/
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
|
||||
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
|
||||
CHECK_ZSTD(dSize);
|
||||
/* When zstd knows the content size, it will error if it doesn't match. */
|
||||
CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
|
||||
|
||||
/* success */
|
||||
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
|
||||
|
||||
ZSTD_freeDCtx(dctx);
|
||||
free(rBuff);
|
||||
free(cBuff);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc<3) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s [FILES] dictionary\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* load dictionary only once */
|
||||
const char* const dictName = argv[argc-1];
|
||||
ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
|
||||
|
||||
int u;
|
||||
for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
|
||||
|
||||
ZSTD_freeDDict(dictPtr);
|
||||
printf("All %u files correctly decoded (in memory) \n", argc-2);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <string.h> // memcpy, strlen
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
typedef struct {
|
||||
void* fBuffer;
|
||||
void* cBuffer;
|
||||
size_t fBufferSize;
|
||||
size_t cBufferSize;
|
||||
ZSTD_CCtx* cctx;
|
||||
} resources;
|
||||
|
||||
/*
|
||||
* allocate memory for buffers big enough to compress all files
|
||||
* as well as memory for output file name (ofn)
|
||||
*/
|
||||
static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen)
|
||||
{
|
||||
size_t maxFilenameLength=0;
|
||||
size_t maxFileSize = 0;
|
||||
|
||||
int argNb;
|
||||
for (argNb = 1; argNb < argc; argNb++) {
|
||||
const char* const filename = argv[argNb];
|
||||
size_t const filenameLength = strlen(filename);
|
||||
size_t const fileSize = fsize_orDie(filename);
|
||||
|
||||
if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
|
||||
if (fileSize > maxFileSize) maxFileSize = fileSize;
|
||||
}
|
||||
|
||||
resources ress;
|
||||
ress.fBufferSize = maxFileSize;
|
||||
ress.cBufferSize = ZSTD_compressBound(maxFileSize);
|
||||
|
||||
*ofnBufferLen = maxFilenameLength + 5;
|
||||
*ofn = (char*)malloc_orDie(*ofnBufferLen);
|
||||
ress.fBuffer = malloc_orDie(ress.fBufferSize);
|
||||
ress.cBuffer = malloc_orDie(ress.cBufferSize);
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
|
||||
return ress;
|
||||
}
|
||||
|
||||
static void freeResources(resources ress, char *outFilename)
|
||||
{
|
||||
free(ress.fBuffer);
|
||||
free(ress.cBuffer);
|
||||
ZSTD_freeCCtx(ress.cctx); /* never fails */
|
||||
free(outFilename);
|
||||
}
|
||||
|
||||
/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
|
||||
static void compressFile_orDie(resources ress, const char* fname, const char* oname)
|
||||
{
|
||||
size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
|
||||
|
||||
/* Compress using the context.
|
||||
* If you need more control over parameters, use the advanced API:
|
||||
* ZSTD_CCtx_setParameter(), and ZSTD_compress2().
|
||||
*/
|
||||
size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
|
||||
CHECK_ZSTD(cSize);
|
||||
|
||||
saveFile_orDie(oname, ress.cBuffer, cSize);
|
||||
|
||||
/* success */
|
||||
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc<2) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s FILE(s)\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* memory allocation for outFilename and resources */
|
||||
char* outFilename;
|
||||
size_t outFilenameBufferLen;
|
||||
resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
|
||||
|
||||
/* compress files with shared context, input and output buffers */
|
||||
int argNb;
|
||||
for (argNb = 1; argNb < argc; argNb++) {
|
||||
const char* const inFilename = argv[argNb];
|
||||
size_t const inFilenameLen = strlen(inFilename);
|
||||
CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
|
||||
memcpy(outFilename, inFilename, inFilenameLen);
|
||||
memcpy(outFilename+inFilenameLen, ".zst", 5);
|
||||
compressFile_orDie(ress, inFilename, outFilename);
|
||||
}
|
||||
|
||||
/* free memory */
|
||||
freeResources(ress,outFilename);
|
||||
|
||||
printf("compressed %i files \n", argc-1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/* The objective of this example is to show of to compress multiple successive files
|
||||
* while preserving memory management.
|
||||
* All structures and buffers will be created only once,
|
||||
* and shared across all compression operations */
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <string.h> // memset, strcat
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
typedef struct {
|
||||
void* buffIn;
|
||||
void* buffOut;
|
||||
size_t buffInSize;
|
||||
size_t buffOutSize;
|
||||
ZSTD_CCtx* cctx;
|
||||
} resources;
|
||||
|
||||
static resources createResources_orDie(int cLevel)
|
||||
{
|
||||
resources ress;
|
||||
ress.buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */
|
||||
ress.buffOutSize= ZSTD_CStreamOutSize(); /* can always flush a full block */
|
||||
ress.buffIn = malloc_orDie(ress.buffInSize);
|
||||
ress.buffOut= malloc_orDie(ress.buffOutSize);
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
|
||||
|
||||
/* Set any compression parameters you want here.
|
||||
* They will persist for every compression operation.
|
||||
* Here we set the compression level, and enable the checksum.
|
||||
*/
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) );
|
||||
return ress;
|
||||
}
|
||||
|
||||
static void freeResources(resources ress)
|
||||
{
|
||||
ZSTD_freeCCtx(ress.cctx);
|
||||
free(ress.buffIn);
|
||||
free(ress.buffOut);
|
||||
}
|
||||
|
||||
static void compressFile_orDie(resources ress, const char* fname, const char* outName)
|
||||
{
|
||||
// Open the input and output files.
|
||||
FILE* const fin = fopen_orDie(fname, "rb");
|
||||
FILE* const fout = fopen_orDie(outName, "wb");
|
||||
|
||||
/* Reset the context to a clean state to start a new compression operation.
|
||||
* The parameters are sticky, so we keep the compression level and extra
|
||||
* parameters that we set in createResources_orDie().
|
||||
*/
|
||||
CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) );
|
||||
|
||||
size_t const toRead = ress.buffInSize;
|
||||
size_t read;
|
||||
while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
|
||||
/* This loop is the same as streaming_compression.c.
|
||||
* See that file for detailed comments.
|
||||
*/
|
||||
int const lastChunk = (read < toRead);
|
||||
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
|
||||
|
||||
ZSTD_inBuffer input = { ress.buffIn, read, 0 };
|
||||
int finished;
|
||||
do {
|
||||
ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
|
||||
size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
|
||||
CHECK_ZSTD(remaining);
|
||||
fwrite_orDie(ress.buffOut, output.pos, fout);
|
||||
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
|
||||
} while (!finished);
|
||||
CHECK(input.pos == input.size,
|
||||
"Impossible: zstd only returns 0 when the input is completely consumed!");
|
||||
}
|
||||
|
||||
fclose_orDie(fout);
|
||||
fclose_orDie(fin);
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc<2) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s FILE(s)\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int const cLevel = 7;
|
||||
resources const ress = createResources_orDie(cLevel);
|
||||
void* ofnBuffer = NULL;
|
||||
size_t ofnbSize = 0;
|
||||
|
||||
int argNb;
|
||||
for (argNb = 1; argNb < argc; argNb++) {
|
||||
const char* const ifn = argv[argNb];
|
||||
size_t const ifnSize = strlen(ifn);
|
||||
size_t const ofnSize = ifnSize + 5;
|
||||
if (ofnbSize <= ofnSize) {
|
||||
ofnbSize = ofnSize + 16;
|
||||
free(ofnBuffer);
|
||||
ofnBuffer = malloc_orDie(ofnbSize);
|
||||
}
|
||||
memset(ofnBuffer, 0, ofnSize);
|
||||
strcat(ofnBuffer, ifn);
|
||||
strcat(ofnBuffer, ".zst");
|
||||
compressFile_orDie(ress, ifn, ofnBuffer);
|
||||
}
|
||||
|
||||
freeResources(ress);
|
||||
free(ofnBuffer);
|
||||
|
||||
printf("compressed %i files \n", argc-1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <string.h> // strlen, strcat, memset
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
static void compress_orDie(const char* fname, const char* oname)
|
||||
{
|
||||
size_t fSize;
|
||||
void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
|
||||
size_t const cBuffSize = ZSTD_compressBound(fSize);
|
||||
void* const cBuff = malloc_orDie(cBuffSize);
|
||||
|
||||
/* Compress.
|
||||
* If you are doing many compressions, you may want to reuse the context.
|
||||
* See the multiple_simple_compression.c example.
|
||||
*/
|
||||
size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
|
||||
CHECK_ZSTD(cSize);
|
||||
|
||||
saveFile_orDie(oname, cBuff, cSize);
|
||||
|
||||
/* success */
|
||||
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
|
||||
|
||||
free(fBuff);
|
||||
free(cBuff);
|
||||
}
|
||||
|
||||
static char* createOutFilename_orDie(const char* filename)
|
||||
{
|
||||
size_t const inL = strlen(filename);
|
||||
size_t const outL = inL + 5;
|
||||
void* const outSpace = malloc_orDie(outL);
|
||||
memset(outSpace, 0, outL);
|
||||
strcat(outSpace, filename);
|
||||
strcat(outSpace, ".zst");
|
||||
return (char*)outSpace;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc!=2) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* const inFilename = argv[1];
|
||||
|
||||
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||
compress_orDie(inFilename, outFilename);
|
||||
free(outFilename);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
static void decompress(const char* fname)
|
||||
{
|
||||
size_t cSize;
|
||||
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
|
||||
/* Read the content size from the frame header. For simplicity we require
|
||||
* that it is always present. By default, zstd will write the content size
|
||||
* in the header when it is known. If you can't guarantee that the frame
|
||||
* content size is always written into the header, either use streaming
|
||||
* decompression, or ZSTD_decompressBound().
|
||||
*/
|
||||
unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
|
||||
CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
|
||||
CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
|
||||
|
||||
void* const rBuff = malloc_orDie((size_t)rSize);
|
||||
|
||||
/* Decompress.
|
||||
* If you are doing many decompressions, you may want to reuse the context
|
||||
* and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
|
||||
* use ZSTD_DCtx_setParameter().
|
||||
*/
|
||||
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
|
||||
CHECK_ZSTD(dSize);
|
||||
/* When zstd knows the content size, it will error if it doesn't match. */
|
||||
CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
|
||||
|
||||
/* success */
|
||||
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
|
||||
|
||||
free(rBuff);
|
||||
free(cBuff);
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc!=2) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
decompress(argv[1]);
|
||||
|
||||
printf("%s correctly decoded (in memory). \n", argv[1]);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <string.h> // memset, strcat, strlen
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
static void compressFile_orDie(const char* fname, const char* outName, int cLevel,
|
||||
int nbThreads)
|
||||
{
|
||||
fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n",
|
||||
fname, cLevel, nbThreads);
|
||||
|
||||
/* Open the input and output files. */
|
||||
FILE* const fin = fopen_orDie(fname, "rb");
|
||||
FILE* const fout = fopen_orDie(outName, "wb");
|
||||
/* Create the input and output buffers.
|
||||
* They may be any size, but we recommend using these functions to size them.
|
||||
* Performance will only suffer significantly for very tiny buffers.
|
||||
*/
|
||||
size_t const buffInSize = ZSTD_CStreamInSize();
|
||||
void* const buffIn = malloc_orDie(buffInSize);
|
||||
size_t const buffOutSize = ZSTD_CStreamOutSize();
|
||||
void* const buffOut = malloc_orDie(buffOutSize);
|
||||
|
||||
/* Create the context. */
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
|
||||
|
||||
/* Set any parameters you want.
|
||||
* Here we set the compression level, and enable the checksum.
|
||||
*/
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
|
||||
if (nbThreads > 1) {
|
||||
size_t const r = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
|
||||
if (ZSTD_isError(r)) {
|
||||
fprintf (stderr, "Note: the linked libzstd library doesn't support multithreading. "
|
||||
"Reverting to single-thread mode. \n");
|
||||
}
|
||||
}
|
||||
|
||||
/* This loop read from the input file, compresses that entire chunk,
|
||||
* and writes all output produced to the output file.
|
||||
*/
|
||||
size_t const toRead = buffInSize;
|
||||
for (;;) {
|
||||
size_t read = fread_orDie(buffIn, toRead, fin);
|
||||
/* Select the flush mode.
|
||||
* If the read may not be finished (read == toRead) we use
|
||||
* ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
|
||||
* Zstd optimizes the case where the first flush mode is ZSTD_e_end,
|
||||
* since it knows it is compressing the entire source in one pass.
|
||||
*/
|
||||
int const lastChunk = (read < toRead);
|
||||
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
|
||||
/* Set the input buffer to what we just read.
|
||||
* We compress until the input buffer is empty, each time flushing the
|
||||
* output.
|
||||
*/
|
||||
ZSTD_inBuffer input = { buffIn, read, 0 };
|
||||
int finished;
|
||||
do {
|
||||
/* Compress into the output buffer and write all of the output to
|
||||
* the file so we can reuse the buffer next iteration.
|
||||
*/
|
||||
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
||||
size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
|
||||
CHECK_ZSTD(remaining);
|
||||
fwrite_orDie(buffOut, output.pos, fout);
|
||||
/* If we're on the last chunk we're finished when zstd returns 0,
|
||||
* which means its consumed all the input AND finished the frame.
|
||||
* Otherwise, we're finished when we've consumed all the input.
|
||||
*/
|
||||
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
|
||||
} while (!finished);
|
||||
CHECK(input.pos == input.size,
|
||||
"Impossible: zstd only returns 0 when the input is completely consumed!");
|
||||
|
||||
if (lastChunk) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
fclose_orDie(fout);
|
||||
fclose_orDie(fin);
|
||||
free(buffIn);
|
||||
free(buffOut);
|
||||
}
|
||||
|
||||
|
||||
static char* createOutFilename_orDie(const char* filename)
|
||||
{
|
||||
size_t const inL = strlen(filename);
|
||||
size_t const outL = inL + 5;
|
||||
void* const outSpace = malloc_orDie(outL);
|
||||
memset(outSpace, 0, outL);
|
||||
strcat(outSpace, filename);
|
||||
strcat(outSpace, ".zst");
|
||||
return (char*)outSpace;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc < 2) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s FILE [LEVEL] [THREADS]\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int cLevel = 1;
|
||||
int nbThreads = 1;
|
||||
|
||||
if (argc >= 3) {
|
||||
cLevel = atoi (argv[2]);
|
||||
CHECK(cLevel != 0, "can't parse LEVEL!");
|
||||
}
|
||||
|
||||
if (argc >= 4) {
|
||||
nbThreads = atoi (argv[3]);
|
||||
CHECK(nbThreads != 0, "can't parse THREADS!");
|
||||
}
|
||||
|
||||
const char* const inFilename = argv[1];
|
||||
|
||||
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||
compressFile_orDie(inFilename, outFilename, cLevel, nbThreads);
|
||||
|
||||
free(outFilename); /* not strictly required, since program execution stops there,
|
||||
* but some static analyzer may complain otherwise */
|
||||
return 0;
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Martin Liska, SUSE, Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
#include <stdio.h> // printf
|
||||
#include <stdlib.h> // free
|
||||
#include <string.h> // memset, strcat, strlen
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
#include <pthread.h>
|
||||
|
||||
typedef struct compress_args
|
||||
{
|
||||
const char *fname;
|
||||
char *outName;
|
||||
int cLevel;
|
||||
#if defined(ZSTD_STATIC_LINKING_ONLY)
|
||||
ZSTD_threadPool *pool;
|
||||
#endif
|
||||
} compress_args_t;
|
||||
|
||||
static void *compressFile_orDie(void *data)
|
||||
{
|
||||
const int nbThreads = 16;
|
||||
|
||||
compress_args_t *args = (compress_args_t *)data;
|
||||
fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n", args->fname, args->cLevel, nbThreads);
|
||||
/* Open the input and output files. */
|
||||
FILE* const fin = fopen_orDie(args->fname, "rb");
|
||||
FILE* const fout = fopen_orDie(args->outName, "wb");
|
||||
/* Create the input and output buffers.
|
||||
* They may be any size, but we recommend using these functions to size them.
|
||||
* Performance will only suffer significantly for very tiny buffers.
|
||||
*/
|
||||
size_t const buffInSize = ZSTD_CStreamInSize();
|
||||
void* const buffIn = malloc_orDie(buffInSize);
|
||||
size_t const buffOutSize = ZSTD_CStreamOutSize();
|
||||
void* const buffOut = malloc_orDie(buffOutSize);
|
||||
|
||||
/* Create the context. */
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
|
||||
|
||||
#if defined(ZSTD_STATIC_LINKING_ONLY)
|
||||
size_t r = ZSTD_CCtx_refThreadPool(cctx, args->pool);
|
||||
CHECK(r == 0, "ZSTD_CCtx_refThreadPool failed!");
|
||||
#endif
|
||||
|
||||
/* Set any parameters you want.
|
||||
* Here we set the compression level, and enable the checksum.
|
||||
*/
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) );
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
|
||||
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
|
||||
|
||||
/* This loop reads from the input file, compresses that entire chunk,
|
||||
* and writes all output produced to the output file.
|
||||
*/
|
||||
size_t const toRead = buffInSize;
|
||||
for (;;) {
|
||||
size_t read = fread_orDie(buffIn, toRead, fin);
|
||||
/* Select the flush mode.
|
||||
* If the read may not be finished (read == toRead) we use
|
||||
* ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
|
||||
* Zstd optimizes the case where the first flush mode is ZSTD_e_end,
|
||||
* since it knows it is compressing the entire source in one pass.
|
||||
*/
|
||||
int const lastChunk = (read < toRead);
|
||||
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
|
||||
/* Set the input buffer to what we just read.
|
||||
* We compress until the input buffer is empty, each time flushing the
|
||||
* output.
|
||||
*/
|
||||
ZSTD_inBuffer input = { buffIn, read, 0 };
|
||||
int finished;
|
||||
do {
|
||||
/* Compress into the output buffer and write all of the output to
|
||||
* the file so we can reuse the buffer next iteration.
|
||||
*/
|
||||
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
||||
size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
|
||||
CHECK_ZSTD(remaining);
|
||||
fwrite_orDie(buffOut, output.pos, fout);
|
||||
/* If we're on the last chunk we're finished when zstd returns 0,
|
||||
* which means its consumed all the input AND finished the frame.
|
||||
* Otherwise, we're finished when we've consumed all the input.
|
||||
*/
|
||||
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
|
||||
} while (!finished);
|
||||
CHECK(input.pos == input.size,
|
||||
"Impossible: zstd only returns 0 when the input is completely consumed!");
|
||||
|
||||
if (lastChunk) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
fprintf (stderr, "Finishing compression of %s\n", args->outName);
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
fclose_orDie(fout);
|
||||
fclose_orDie(fin);
|
||||
free(buffIn);
|
||||
free(buffOut);
|
||||
free(args->outName);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
static char* createOutFilename_orDie(const char* filename)
|
||||
{
|
||||
size_t const inL = strlen(filename);
|
||||
size_t const outL = inL + 5;
|
||||
void* const outSpace = malloc_orDie(outL);
|
||||
memset(outSpace, 0, outL);
|
||||
strcat(outSpace, filename);
|
||||
strcat(outSpace, ".zst");
|
||||
return (char*)outSpace;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc<=3) {
|
||||
printf("wrong arguments\n");
|
||||
printf("usage:\n");
|
||||
printf("%s POOL_SIZE LEVEL FILES\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int pool_size = atoi (argv[1]);
|
||||
CHECK(pool_size != 0, "can't parse POOL_SIZE!");
|
||||
|
||||
int level = atoi (argv[2]);
|
||||
CHECK(level != 0, "can't parse LEVEL!");
|
||||
|
||||
argc -= 3;
|
||||
argv += 3;
|
||||
|
||||
#if defined(ZSTD_STATIC_LINKING_ONLY)
|
||||
ZSTD_threadPool *pool = ZSTD_createThreadPool (pool_size);
|
||||
CHECK(pool != NULL, "ZSTD_createThreadPool() failed!");
|
||||
fprintf (stderr, "Using shared thread pool of size %d\n", pool_size);
|
||||
#else
|
||||
fprintf (stderr, "All threads use its own thread pool\n");
|
||||
#endif
|
||||
|
||||
pthread_t *threads = malloc_orDie(argc * sizeof(pthread_t));
|
||||
compress_args_t *args = malloc_orDie(argc * sizeof(compress_args_t));
|
||||
|
||||
for (unsigned i = 0; i < argc; i++)
|
||||
{
|
||||
args[i].fname = argv[i];
|
||||
args[i].outName = createOutFilename_orDie(args[i].fname);
|
||||
args[i].cLevel = level;
|
||||
#if defined(ZSTD_STATIC_LINKING_ONLY)
|
||||
args[i].pool = pool;
|
||||
#endif
|
||||
|
||||
pthread_create (&threads[i], NULL, compressFile_orDie, &args[i]);
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < argc; i++)
|
||||
pthread_join (threads[i], NULL);
|
||||
|
||||
#if defined(ZSTD_STATIC_LINKING_ONLY)
|
||||
ZSTD_freeThreadPool (pool);
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
#include <stdio.h> // fprintf
|
||||
#include <stdlib.h> // free
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
static void decompressFile_orDie(const char* fname)
|
||||
{
|
||||
FILE* const fin = fopen_orDie(fname, "rb");
|
||||
size_t const buffInSize = ZSTD_DStreamInSize();
|
||||
void* const buffIn = malloc_orDie(buffInSize);
|
||||
FILE* const fout = stdout;
|
||||
size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
|
||||
void* const buffOut = malloc_orDie(buffOutSize);
|
||||
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
|
||||
|
||||
/* This loop assumes that the input file is one or more concatenated zstd
|
||||
* streams. This example won't work if there is trailing non-zstd data at
|
||||
* the end, but streaming decompression in general handles this case.
|
||||
* ZSTD_decompressStream() returns 0 exactly when the frame is completed,
|
||||
* and doesn't consume input after the frame.
|
||||
*/
|
||||
size_t const toRead = buffInSize;
|
||||
size_t read;
|
||||
size_t lastRet = 0;
|
||||
int isEmpty = 1;
|
||||
while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
|
||||
isEmpty = 0;
|
||||
ZSTD_inBuffer input = { buffIn, read, 0 };
|
||||
/* Given a valid frame, zstd won't consume the last byte of the frame
|
||||
* until it has flushed all of the decompressed data of the frame.
|
||||
* Therefore, instead of checking if the return code is 0, we can
|
||||
* decompress just check if input.pos < input.size.
|
||||
*/
|
||||
while (input.pos < input.size) {
|
||||
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
||||
/* The return code is zero if the frame is complete, but there may
|
||||
* be multiple frames concatenated together. Zstd will automatically
|
||||
* reset the context when a frame is complete. Still, calling
|
||||
* ZSTD_DCtx_reset() can be useful to reset the context to a clean
|
||||
* state, for instance if the last decompression call returned an
|
||||
* error.
|
||||
*/
|
||||
size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
|
||||
CHECK_ZSTD(ret);
|
||||
fwrite_orDie(buffOut, output.pos, fout);
|
||||
lastRet = ret;
|
||||
}
|
||||
}
|
||||
|
||||
if (isEmpty) {
|
||||
fprintf(stderr, "input is empty\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (lastRet != 0) {
|
||||
/* The last return value from ZSTD_decompressStream did not end on a
|
||||
* frame, but we reached the end of the file! We assume this is an
|
||||
* error, and the input was truncated.
|
||||
*/
|
||||
fprintf(stderr, "EOF before end of stream: %zu\n", lastRet);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
ZSTD_freeDCtx(dctx);
|
||||
fclose_orDie(fin);
|
||||
fclose_orDie(fout);
|
||||
free(buffIn);
|
||||
free(buffOut);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
|
||||
if (argc!=2) {
|
||||
fprintf(stderr, "wrong arguments\n");
|
||||
fprintf(stderr, "usage:\n");
|
||||
fprintf(stderr, "%s FILE\n", exeName);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char* const inFilename = argv[1];
|
||||
|
||||
decompressFile_orDie(inFilename);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/*=== Tuning parameter ===*/
|
||||
#ifndef MAX_TESTED_LEVEL
|
||||
#define MAX_TESTED_LEVEL 12
|
||||
#endif
|
||||
|
||||
|
||||
/*=== Dependencies ===*/
|
||||
#include <stdio.h> // printf
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
#include "common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
|
||||
|
||||
|
||||
/*=== functions ===*/
|
||||
|
||||
/*! readU32FromChar() :
|
||||
@return : unsigned integer value read from input in `char` format
|
||||
allows and interprets K, KB, KiB, M, MB and MiB suffix.
|
||||
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
Note : function result can overflow if digit string > MAX_UINT */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9'))
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
if ((**stringPtr=='K') || (**stringPtr=='M')) {
|
||||
result <<= 10;
|
||||
if (**stringPtr=='M') result <<= 10;
|
||||
(*stringPtr)++ ;
|
||||
if (**stringPtr=='i') (*stringPtr)++;
|
||||
if (**stringPtr=='B') (*stringPtr)++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char const *argv[]) {
|
||||
|
||||
printf("\n Zstandard (v%s) memory usage for streaming : \n\n", ZSTD_versionString());
|
||||
|
||||
unsigned wLog = 0;
|
||||
if (argc > 1) {
|
||||
const char* valStr = argv[1];
|
||||
wLog = readU32FromChar(&valStr);
|
||||
}
|
||||
|
||||
int compressionLevel;
|
||||
for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) {
|
||||
#define INPUT_SIZE 5
|
||||
#define COMPRESSED_SIZE 128
|
||||
char const dataToCompress[INPUT_SIZE] = "abcde";
|
||||
char compressedData[COMPRESSED_SIZE];
|
||||
char decompressedData[INPUT_SIZE];
|
||||
/* the ZSTD_CCtx_params structure is a way to save parameters and use
|
||||
* them across multiple contexts. We use them here so we can call the
|
||||
* function ZSTD_estimateCStreamSize_usingCCtxParams().
|
||||
*/
|
||||
ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
|
||||
CHECK(cctxParams != NULL, "ZSTD_createCCtxParams() failed!");
|
||||
|
||||
/* Set the compression level. */
|
||||
CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, compressionLevel) );
|
||||
/* Set the window log.
|
||||
* The value 0 means use the default window log, which is equivalent to
|
||||
* not setting it.
|
||||
*/
|
||||
CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, wLog) );
|
||||
|
||||
/* Force the compressor to allocate the maximum memory size for a given
|
||||
* level by not providing the pledged source size, or calling
|
||||
* ZSTD_compressStream2() with ZSTD_e_end.
|
||||
*/
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
|
||||
CHECK_ZSTD( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
|
||||
size_t compressedSize;
|
||||
{
|
||||
ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
|
||||
ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
|
||||
CHECK_ZSTD( ZSTD_compressStream(cctx, &outBuff, &inBuff) );
|
||||
size_t const remaining = ZSTD_endStream(cctx, &outBuff);
|
||||
CHECK_ZSTD(remaining);
|
||||
CHECK(remaining == 0, "Frame not flushed!");
|
||||
compressedSize = outBuff.pos;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
|
||||
/* Set the maximum allowed window log.
|
||||
* The value 0 means use the default window log, which is equivalent to
|
||||
* not setting it.
|
||||
*/
|
||||
CHECK_ZSTD( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, wLog) );
|
||||
/* forces decompressor to use maximum memory size, since the
|
||||
* decompressed size is not stored in the frame header.
|
||||
*/
|
||||
{ ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
|
||||
ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
|
||||
size_t const remaining = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
|
||||
CHECK_ZSTD(remaining);
|
||||
CHECK(remaining == 0, "Frame not complete!");
|
||||
CHECK(outBuff.pos == sizeof(dataToCompress), "Bad decompression!");
|
||||
}
|
||||
|
||||
size_t const cstreamSize = ZSTD_sizeof_CStream(cctx);
|
||||
size_t const cstreamEstimatedSize = ZSTD_estimateCStreamSize_usingCCtxParams(cctxParams);
|
||||
size_t const dstreamSize = ZSTD_sizeof_DStream(dctx);
|
||||
size_t const dstreamEstimatedSize = ZSTD_estimateDStreamSize_fromFrame(compressedData, compressedSize);
|
||||
|
||||
CHECK(cstreamSize <= cstreamEstimatedSize, "Compression mem (%u) > estimated (%u)",
|
||||
(unsigned)cstreamSize, (unsigned)cstreamEstimatedSize);
|
||||
CHECK(dstreamSize <= dstreamEstimatedSize, "Decompression mem (%u) > estimated (%u)",
|
||||
(unsigned)dstreamSize, (unsigned)dstreamEstimatedSize);
|
||||
|
||||
printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB (estimated : %5u KB)\n",
|
||||
compressionLevel,
|
||||
(unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10),
|
||||
(unsigned)(dstreamSize>>10), (unsigned)(dstreamEstimatedSize>>10));
|
||||
|
||||
ZSTD_freeDCtx(dctx);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeCCtxParams(cctxParams);
|
||||
if (wLog) break; /* single test */
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -1,232 +0,0 @@
|
||||
cxx_library(
|
||||
name='zstd',
|
||||
header_namespace='',
|
||||
exported_headers=['zstd.h'],
|
||||
visibility=['PUBLIC'],
|
||||
deps=[
|
||||
':common',
|
||||
':compress',
|
||||
':decompress',
|
||||
':deprecated',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='compress',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('compress', 'zstd*.h'),
|
||||
]),
|
||||
srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
|
||||
deps=[':common'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='decompress',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
headers=subdir_glob([
|
||||
('decompress', '*_impl.h'),
|
||||
]),
|
||||
srcs=glob(['decompress/zstd*.c']),
|
||||
deps=[
|
||||
':common',
|
||||
':legacy',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='deprecated',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('deprecated', '*.h'),
|
||||
]),
|
||||
srcs=glob(['deprecated/*.c']),
|
||||
deps=[':common'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='legacy',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('legacy', '*.h'),
|
||||
]),
|
||||
srcs=glob(['legacy/*.c']),
|
||||
deps=[':common'],
|
||||
exported_preprocessor_flags=[
|
||||
'-DZSTD_LEGACY_SUPPORT=4',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='zdict',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=['zdict.h'],
|
||||
headers=subdir_glob([
|
||||
('dictBuilder', 'divsufsort.h'),
|
||||
('dictBuilder', 'cover.h'),
|
||||
]),
|
||||
srcs=glob(['dictBuilder/*.c']),
|
||||
deps=[':common'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='compiler',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'compiler.h'),
|
||||
]),
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='cpu',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'cpu.h'),
|
||||
]),
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='bitstream',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'bitstream.h'),
|
||||
]),
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='entropy',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'fse.h'),
|
||||
('common', 'huf.h'),
|
||||
]),
|
||||
srcs=[
|
||||
'common/entropy_common.c',
|
||||
'common/fse_decompress.c',
|
||||
'compress/fse_compress.c',
|
||||
'compress/huf_compress.c',
|
||||
'decompress/huf_decompress.c',
|
||||
],
|
||||
deps=[
|
||||
':debug',
|
||||
':bitstream',
|
||||
':compiler',
|
||||
':errors',
|
||||
':mem',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='errors',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=[
|
||||
'zstd_errors.h',
|
||||
'common/error_private.h',
|
||||
]
|
||||
srcs=['common/error_private.c'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='mem',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'mem.h'),
|
||||
]),
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='pool',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'pool.h'),
|
||||
]),
|
||||
srcs=['common/pool.c'],
|
||||
deps=[
|
||||
':threading',
|
||||
':zstd_common',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='threading',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'threading.h'),
|
||||
]),
|
||||
srcs=['common/threading.c'],
|
||||
exported_preprocessor_flags=[
|
||||
'-DZSTD_MULTITHREAD',
|
||||
],
|
||||
exported_linker_flags=[
|
||||
'-pthread',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='xxhash',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'xxhash.h'),
|
||||
]),
|
||||
srcs=['common/xxhash.c'],
|
||||
exported_preprocessor_flags=[
|
||||
'-DXXH_NAMESPACE=ZSTD_',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='zstd_common',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('', 'zstd.h'),
|
||||
('common', 'zstd_internal.h'),
|
||||
]),
|
||||
srcs=['common/zstd_common.c'],
|
||||
deps=[
|
||||
':compiler',
|
||||
':errors',
|
||||
':mem',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='debug',
|
||||
header_namespace='',
|
||||
visibility=['PUBLIC'],
|
||||
exported_headers=subdir_glob([
|
||||
('common', 'debug.h'),
|
||||
]),
|
||||
srcs=['common/debug.c'],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='common',
|
||||
deps=[
|
||||
':debug',
|
||||
':bitstream',
|
||||
':compiler',
|
||||
':cpu',
|
||||
':entropy',
|
||||
':errors',
|
||||
':mem',
|
||||
':pool',
|
||||
':threading',
|
||||
':xxhash',
|
||||
':zstd_common',
|
||||
]
|
||||
)
|
||||
@@ -1,389 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
# default target (when running `make` with no argument)
|
||||
lib-release:
|
||||
|
||||
# Modules
|
||||
ZSTD_LIB_COMPRESSION ?= 1
|
||||
ZSTD_LIB_DECOMPRESSION ?= 1
|
||||
ZSTD_LIB_DICTBUILDER ?= 1
|
||||
ZSTD_LIB_DEPRECATED ?= 0
|
||||
|
||||
# Input variables for libzstd.mk
|
||||
ifeq ($(ZSTD_LIB_COMPRESSION), 0)
|
||||
ZSTD_LIB_DICTBUILDER = 0
|
||||
ZSTD_LIB_DEPRECATED = 0
|
||||
endif
|
||||
|
||||
ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
|
||||
ZSTD_LEGACY_SUPPORT = 0
|
||||
ZSTD_LIB_DEPRECATED = 0
|
||||
endif
|
||||
|
||||
include libzstd.mk
|
||||
|
||||
ZSTD_FILES := $(ZSTD_COMMON_FILES) $(ZSTD_LEGACY_FILES)
|
||||
|
||||
ifneq ($(ZSTD_LIB_COMPRESSION), 0)
|
||||
ZSTD_FILES += $(ZSTD_COMPRESS_FILES)
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
|
||||
ZSTD_FILES += $(ZSTD_DECOMPRESS_FILES)
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_LIB_DEPRECATED), 0)
|
||||
ZSTD_FILES += $(ZSTD_DEPRECATED_FILES)
|
||||
endif
|
||||
|
||||
ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
|
||||
ZSTD_FILES += $(ZSTD_DICTBUILDER_FILES)
|
||||
endif
|
||||
|
||||
ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
|
||||
ZSTD_LOCAL_OBJ0 := $(ZSTD_LOCAL_SRC:.c=.o)
|
||||
ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_OBJ0:.S=.o)
|
||||
|
||||
VERSION := $(ZSTD_VERSION)
|
||||
|
||||
# Note: by default, the static library is built single-threaded and dynamic library is built
|
||||
# multi-threaded. It is possible to force multi or single threaded builds by appending
|
||||
# -mt or -nomt to the build target (like lib-mt for multi-threaded, lib-nomt for single-threaded).
|
||||
|
||||
|
||||
CPPFLAGS_DYNLIB += -DZSTD_MULTITHREAD # dynamic library build defaults to multi-threaded
|
||||
LDFLAGS_DYNLIB += -pthread
|
||||
CPPFLAGS_STATICLIB += # static library build defaults to single-threaded
|
||||
|
||||
# pkg-config Libs.private points to LDFLAGS_DYNLIB
|
||||
PCLIB := $(LDFLAGS_DYNLIB)
|
||||
|
||||
ifeq ($(findstring GCC,$(CCVER)),GCC)
|
||||
decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
|
||||
endif
|
||||
|
||||
|
||||
# macOS linker doesn't support -soname, and use different extension
|
||||
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
|
||||
UNAME_TARGET_SYSTEM ?= $(UNAME)
|
||||
|
||||
ifeq ($(UNAME_TARGET_SYSTEM), Darwin)
|
||||
SHARED_EXT = dylib
|
||||
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
|
||||
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
|
||||
SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
|
||||
else
|
||||
ifeq ($(UNAME_TARGET_SYSTEM), AIX)
|
||||
SONAME_FLAGS =
|
||||
else
|
||||
SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
|
||||
endif
|
||||
SHARED_EXT = so
|
||||
SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
|
||||
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
|
||||
endif
|
||||
|
||||
|
||||
.PHONY: all
|
||||
all: lib
|
||||
|
||||
|
||||
.PHONY: libzstd.a # must be run every time
|
||||
libzstd.a: CPPFLAGS += $(CPPFLAGS_STATICLIB)
|
||||
|
||||
SET_CACHE_DIRECTORY = \
|
||||
+$(MAKE) --no-print-directory $@ \
|
||||
BUILD_DIR=obj/$(HASH_DIR) \
|
||||
CPPFLAGS="$(CPPFLAGS)" \
|
||||
CFLAGS="$(CFLAGS)" \
|
||||
LDFLAGS="$(LDFLAGS)"
|
||||
|
||||
ifndef BUILD_DIR
|
||||
# determine BUILD_DIR from compilation flags
|
||||
|
||||
libzstd.a:
|
||||
$(SET_CACHE_DIRECTORY)
|
||||
|
||||
else
|
||||
# BUILD_DIR is defined
|
||||
|
||||
ZSTD_STATICLIB_DIR := $(BUILD_DIR)/static
|
||||
ZSTD_STATICLIB := $(ZSTD_STATICLIB_DIR)/libzstd.a
|
||||
ZSTD_STATICLIB_OBJ := $(addprefix $(ZSTD_STATICLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
|
||||
$(ZSTD_STATICLIB): ARFLAGS = rcs
|
||||
$(ZSTD_STATICLIB): | $(ZSTD_STATICLIB_DIR)
|
||||
$(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ)
|
||||
# Check for multithread flag at target execution time
|
||||
$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
|
||||
@echo compiling multi-threaded static library $(LIBVER),\
|
||||
@echo compiling single-threaded static library $(LIBVER))
|
||||
$(AR) $(ARFLAGS) $@ $^
|
||||
|
||||
libzstd.a: $(ZSTD_STATICLIB)
|
||||
cp -f $< $@
|
||||
|
||||
endif
|
||||
|
||||
ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
|
||||
|
||||
LIBZSTD = dll/libzstd.dll
|
||||
$(LIBZSTD): $(ZSTD_FILES)
|
||||
@echo compiling dynamic library $(LIBVER)
|
||||
$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll/libzstd.dll.a -shared $^ -o $@
|
||||
|
||||
else # not Windows
|
||||
|
||||
LIBZSTD = libzstd.$(SHARED_EXT_VER)
|
||||
.PHONY: $(LIBZSTD) # must be run every time
|
||||
$(LIBZSTD): CPPFLAGS += $(CPPFLAGS_DYNLIB)
|
||||
$(LIBZSTD): CFLAGS += -fPIC -fvisibility=hidden
|
||||
$(LIBZSTD): LDFLAGS += -shared $(LDFLAGS_DYNLIB)
|
||||
|
||||
ifndef BUILD_DIR
|
||||
# determine BUILD_DIR from compilation flags
|
||||
|
||||
$(LIBZSTD):
|
||||
$(SET_CACHE_DIRECTORY)
|
||||
|
||||
else
|
||||
# BUILD_DIR is defined
|
||||
|
||||
ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynamic
|
||||
ZSTD_DYNLIB := $(ZSTD_DYNLIB_DIR)/$(LIBZSTD)
|
||||
ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/,$(ZSTD_LOCAL_OBJ))
|
||||
|
||||
$(ZSTD_DYNLIB): | $(ZSTD_DYNLIB_DIR)
|
||||
$(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
|
||||
# Check for multithread flag at target execution time
|
||||
$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
|
||||
@echo compiling multi-threaded dynamic library $(LIBVER),\
|
||||
@echo compiling single-threaded dynamic library $(LIBVER))
|
||||
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
||||
@echo creating versioned links
|
||||
ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
|
||||
ln -sf $@ libzstd.$(SHARED_EXT)
|
||||
|
||||
$(LIBZSTD): $(ZSTD_DYNLIB)
|
||||
cp -f $< $@
|
||||
|
||||
endif # ifndef BUILD_DIR
|
||||
endif # if windows
|
||||
|
||||
.PHONY: libzstd
|
||||
libzstd : $(LIBZSTD)
|
||||
|
||||
.PHONY: lib
|
||||
lib : libzstd.a libzstd
|
||||
|
||||
|
||||
# note : do not define lib-mt or lib-release as .PHONY
|
||||
# make does not consider implicit pattern rule for .PHONY target
|
||||
|
||||
%-mt : CPPFLAGS_DYNLIB := -DZSTD_MULTITHREAD
|
||||
%-mt : CPPFLAGS_STATICLIB := -DZSTD_MULTITHREAD
|
||||
%-mt : LDFLAGS_DYNLIB := -pthread
|
||||
%-mt : PCLIB :=
|
||||
%-mt : PCMTLIB := $(LDFLAGS_DYNLIB)
|
||||
%-mt : %
|
||||
@echo multi-threaded build completed
|
||||
|
||||
%-nomt : CPPFLAGS_DYNLIB :=
|
||||
%-nomt : LDFLAGS_DYNLIB :=
|
||||
%-nomt : CPPFLAGS_STATICLIB :=
|
||||
%-nomt : PCLIB :=
|
||||
%-nomt : %
|
||||
@echo single-threaded build completed
|
||||
|
||||
%-release : DEBUGFLAGS :=
|
||||
%-release : %
|
||||
@echo release build completed
|
||||
|
||||
|
||||
# Generate .h dependencies automatically
|
||||
|
||||
# -MMD: compiler generates dependency information as a side-effect of compilation, without system headers
|
||||
# -MP: adds phony target for each dependency other than main file.
|
||||
DEPFLAGS = -MMD -MP
|
||||
|
||||
# ensure that ZSTD_DYNLIB_DIR exists prior to generating %.o
|
||||
$(ZSTD_DYNLIB_DIR)/%.o : %.c | $(ZSTD_DYNLIB_DIR)
|
||||
@echo CC $@
|
||||
$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
|
||||
|
||||
$(ZSTD_STATICLIB_DIR)/%.o : %.c | $(ZSTD_STATICLIB_DIR)
|
||||
@echo CC $@
|
||||
$(COMPILE.c) $(DEPFLAGS) $(OUTPUT_OPTION) $<
|
||||
|
||||
$(ZSTD_DYNLIB_DIR)/%.o : %.S | $(ZSTD_DYNLIB_DIR)
|
||||
@echo AS $@
|
||||
$(COMPILE.S) $(OUTPUT_OPTION) $<
|
||||
|
||||
$(ZSTD_STATICLIB_DIR)/%.o : %.S | $(ZSTD_STATICLIB_DIR)
|
||||
@echo AS $@
|
||||
$(COMPILE.S) $(OUTPUT_OPTION) $<
|
||||
|
||||
MKDIR ?= mkdir -p
|
||||
$(BUILD_DIR) $(ZSTD_DYNLIB_DIR) $(ZSTD_STATICLIB_DIR):
|
||||
$(MKDIR) $@
|
||||
|
||||
DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATICLIB_OBJ:.o=.d)
|
||||
$(DEPFILES):
|
||||
|
||||
# The leading '-' means: do not fail is include fails (ex: directory does not exist yet)
|
||||
-include $(wildcard $(DEPFILES))
|
||||
|
||||
|
||||
# Special case : build library in single-thread mode _and_ without zstdmt_compress.c
|
||||
# Note : we still need threading.c and pool.c for the dictionary builder,
|
||||
# but they will correctly behave single-threaded.
|
||||
ZSTDMT_FILES = zstdmt_compress.c
|
||||
ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(notdir $(ZSTD_FILES)))
|
||||
libzstd-nomt: CFLAGS += -fPIC -fvisibility=hidden
|
||||
libzstd-nomt: LDFLAGS += -shared
|
||||
libzstd-nomt: $(ZSTD_NOMT_FILES)
|
||||
@echo compiling single-thread dynamic library $(LIBVER)
|
||||
@echo files : $(ZSTD_NOMT_FILES)
|
||||
@if echo "$(ZSTD_NOMT_FILES)" | tr ' ' '\n' | $(GREP) -q zstdmt; then \
|
||||
echo "Error: Found zstdmt in list."; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) -r *.dSYM # macOS-specific
|
||||
$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
|
||||
$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
|
||||
$(RM) -r obj/*
|
||||
@echo Cleaning library completed
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# make install is validated only for below listed environments
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT%,$(UNAME)))
|
||||
|
||||
lib: libzstd.pc
|
||||
|
||||
HAS_EXPLICIT_EXEC_PREFIX := $(if $(or $(EXEC_PREFIX),$(exec_prefix)),1,)
|
||||
|
||||
DESTDIR ?=
|
||||
# directory variables : GNU conventions prefer lowercase
|
||||
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
|
||||
# support both lower and uppercase (BSD), use uppercase in script
|
||||
prefix ?= /usr/local
|
||||
PREFIX ?= $(prefix)
|
||||
exec_prefix ?= $(PREFIX)
|
||||
EXEC_PREFIX ?= $(exec_prefix)
|
||||
libdir ?= $(EXEC_PREFIX)/lib
|
||||
LIBDIR ?= $(libdir)
|
||||
includedir ?= $(PREFIX)/include
|
||||
INCLUDEDIR ?= $(includedir)
|
||||
|
||||
PCINCDIR := $(patsubst $(PREFIX)%,%,$(INCLUDEDIR))
|
||||
PCLIBDIR := $(patsubst $(EXEC_PREFIX)%,%,$(LIBDIR))
|
||||
|
||||
# If we successfully stripped off a prefix, we'll add a reference to the
|
||||
# relevant pc variable.
|
||||
PCINCPREFIX := $(if $(findstring $(INCLUDEDIR),$(PCINCDIR)),,$${prefix})
|
||||
PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
|
||||
|
||||
# If no explicit EXEC_PREFIX was set by the caller, write it out as a reference
|
||||
# to PREFIX, rather than as a resolved value.
|
||||
PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})
|
||||
|
||||
|
||||
ifneq ($(MT),)
|
||||
PCLIB :=
|
||||
PCMTLIB := $(LDFLAGS_DYNLIB)
|
||||
else
|
||||
PCLIB := $(LDFLAGS_DYNLIB)
|
||||
endif
|
||||
|
||||
ifneq (,$(filter FreeBSD NetBSD DragonFly,$(UNAME)))
|
||||
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
|
||||
else
|
||||
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
|
||||
endif
|
||||
|
||||
ifneq (,$(filter SunOS,$(UNAME)))
|
||||
INSTALL ?= ginstall
|
||||
else
|
||||
INSTALL ?= install
|
||||
endif
|
||||
|
||||
INSTALL_PROGRAM ?= $(INSTALL)
|
||||
INSTALL_DATA ?= $(INSTALL) -m 644
|
||||
|
||||
|
||||
# pkg-config library define.
|
||||
# For static single-threaded library declare -pthread in Libs.private
|
||||
# For static multi-threaded library declare -pthread in Libs and Cflags
|
||||
.PHONY: libzstd.pc
|
||||
libzstd.pc: libzstd.pc.in
|
||||
@echo creating pkgconfig
|
||||
@sed \
|
||||
-e 's|@PREFIX@|$(PREFIX)|' \
|
||||
-e 's|@EXEC_PREFIX@|$(PCEXEC_PREFIX)|' \
|
||||
-e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
|
||||
-e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
|
||||
-e 's|@VERSION@|$(VERSION)|' \
|
||||
-e 's|@LIBS_MT@|$(PCMTLIB)|' \
|
||||
-e 's|@LIBS_PRIVATE@|$(PCLIB)|' \
|
||||
$< >$@
|
||||
|
||||
.PHONY: install
|
||||
install: install-pc install-static install-shared install-includes
|
||||
@echo zstd static and shared library installed
|
||||
|
||||
.PHONY: install-pc
|
||||
install-pc: libzstd.pc
|
||||
[ -e $(DESTDIR)$(PKGCONFIGDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
|
||||
$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
|
||||
|
||||
.PHONY: install-static
|
||||
install-static:
|
||||
# only generate libzstd.a if it's not already present
|
||||
[ -e libzstd.a ] || $(MAKE) libzstd.a-release
|
||||
[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
|
||||
@echo Installing static library
|
||||
$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
|
||||
|
||||
.PHONY: install-shared
|
||||
install-shared:
|
||||
# only generate libzstd.so if it's not already present
|
||||
[ -e $(LIBZSTD) ] || $(MAKE) libzstd-release
|
||||
[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
|
||||
@echo Installing shared library
|
||||
$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
|
||||
ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
|
||||
ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
|
||||
|
||||
.PHONY: install-includes
|
||||
install-includes:
|
||||
[ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
|
||||
@echo Installing includes
|
||||
$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
|
||||
$(INSTALL_DATA) zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
|
||||
$(INSTALL_DATA) zdict.h $(DESTDIR)$(INCLUDEDIR)
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
|
||||
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
|
||||
$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
|
||||
$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
|
||||
$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
|
||||
$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
|
||||
$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
|
||||
$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
|
||||
@echo zstd libraries successfully uninstalled
|
||||
|
||||
endif
|
||||
@@ -1,44 +0,0 @@
|
||||
cxx_binary(
|
||||
name='zstd',
|
||||
headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
|
||||
srcs=glob(['*.c'], excludes=['datagen.c']),
|
||||
deps=[
|
||||
':datagen',
|
||||
':util',
|
||||
'//lib:zstd',
|
||||
'//lib:zdict',
|
||||
'//lib:mem',
|
||||
'//lib:xxhash',
|
||||
],
|
||||
preprocessor_flags=[
|
||||
'-DZSTD_GZCOMPRESS',
|
||||
'-DZSTD_GZDECOMPRESS',
|
||||
'-DZSTD_LZMACOMPRESS',
|
||||
'-DZSTD_LZMADECOMPRES',
|
||||
'-DZSTD_LZ4COMPRESS',
|
||||
'-DZSTD_LZ4DECOMPRES',
|
||||
],
|
||||
linker_flags=[
|
||||
'-lz',
|
||||
'-llzma',
|
||||
'-llz4',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_library(
|
||||
name='datagen',
|
||||
visibility=['PUBLIC'],
|
||||
header_namespace='',
|
||||
exported_headers=['datagen.h'],
|
||||
srcs=['datagen.c'],
|
||||
deps=['//lib:mem'],
|
||||
)
|
||||
|
||||
|
||||
cxx_library(
|
||||
name='util',
|
||||
visibility=['PUBLIC'],
|
||||
header_namespace='',
|
||||
exported_headers=['util.h', 'platform.h'],
|
||||
deps=['//lib:mem'],
|
||||
)
|
||||
@@ -1,445 +0,0 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ##########################################################################
|
||||
# zstd : Command Line Utility, supporting gzip-like arguments
|
||||
# zstd32 : Same as zstd, but forced to compile in 32-bits mode
|
||||
# zstd-nolegacy : zstd without support of decompression of legacy versions
|
||||
# zstd-small : minimal zstd without dictionary builder and benchmark
|
||||
# zstd-compress : compressor-only version of zstd
|
||||
# zstd-decompress : decompressor-only version of zstd
|
||||
# ##########################################################################
|
||||
|
||||
# default target (when running `make` with no argument)
|
||||
zstd-release:
|
||||
|
||||
LIBZSTD_MK_DIR = ../lib
|
||||
include $(LIBZSTD_MK_DIR)/libzstd.mk
|
||||
|
||||
ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
|
||||
ALIGN_LOOP = -falign-loops=32
|
||||
else
|
||||
ALIGN_LOOP =
|
||||
endif
|
||||
|
||||
ZSTDLIB_COMMON_SRC := $(sort $(ZSTD_COMMON_FILES))
|
||||
ZSTDLIB_COMPRESS_SRC := $(sort $(ZSTD_COMPRESS_FILES))
|
||||
ZSTDLIB_DECOMPRESS_SRC := $(sort $(ZSTD_DECOMPRESS_FILES))
|
||||
ZSTDLIB_CORE_SRC := $(sort $(ZSTD_DECOMPRESS_FILES) $(ZSTD_COMMON_FILES) $(ZSTD_COMPRESS_FILES))
|
||||
ZDICT_SRC := $(sort $(ZSTD_DICTBUILDER_FILES))
|
||||
ZSTDLEGACY_SRC := $(sort $(ZSTD_LEGACY_FILES))
|
||||
|
||||
# Sort files in alphabetical order for reproducible builds
|
||||
ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
|
||||
ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC))
|
||||
ZSTDLIB_LOCAL_OBJ0 := $(ZSTDLIB_LOCAL_SRC:.c=.o)
|
||||
ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_OBJ0:.S=.o)
|
||||
|
||||
ZSTD_CLI_SRC := $(sort $(wildcard *.c))
|
||||
ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
|
||||
|
||||
ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
|
||||
ZSTD_ALL_OBJ0 := $(ZSTD_ALL_SRC:.c=.o)
|
||||
ZSTD_ALL_OBJ := $(ZSTD_ALL_OBJ0:.S=.o)
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
RES64_FILE = windres/zstd64.res
|
||||
RES32_FILE = windres/zstd32.res
|
||||
ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
|
||||
RES_FILE = $(RES64_FILE)
|
||||
else
|
||||
RES_FILE = $(RES32_FILE)
|
||||
endif
|
||||
else
|
||||
EXT =
|
||||
endif
|
||||
|
||||
# thread detection
|
||||
NO_THREAD_MSG := ==> no threads, building without multithreading support
|
||||
HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
|
||||
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
|
||||
ifeq ($(HAVE_THREAD), 1)
|
||||
THREAD_MSG := ==> building with threading support
|
||||
THREAD_CPP := -DZSTD_MULTITHREAD
|
||||
THREAD_LD := -pthread
|
||||
else
|
||||
THREAD_MSG := $(NO_THREAD_MSG)
|
||||
endif
|
||||
|
||||
# zlib detection
|
||||
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
|
||||
HAVE_ZLIB ?= $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
|
||||
ifeq ($(HAVE_ZLIB), 1)
|
||||
ZLIB_MSG := ==> building zstd with .gz compression support
|
||||
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
|
||||
ZLIBLD = -lz
|
||||
else
|
||||
ZLIB_MSG := $(NO_ZLIB_MSG)
|
||||
endif
|
||||
|
||||
# lzma detection
|
||||
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
|
||||
HAVE_LZMA ?= $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
|
||||
ifeq ($(HAVE_LZMA), 1)
|
||||
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
|
||||
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
|
||||
LZMALD = -llzma
|
||||
else
|
||||
LZMA_MSG := $(NO_LZMA_MSG)
|
||||
endif
|
||||
|
||||
# lz4 detection
|
||||
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
|
||||
HAVE_LZ4 ?= $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
|
||||
ifeq ($(HAVE_LZ4), 1)
|
||||
LZ4_MSG := ==> building zstd with .lz4 compression support
|
||||
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
|
||||
LZ4LD = -llz4
|
||||
else
|
||||
LZ4_MSG := $(NO_LZ4_MSG)
|
||||
endif
|
||||
|
||||
# explicit backtrace enable/disable for Linux & Darwin
|
||||
ifeq ($(BACKTRACE), 0)
|
||||
DEBUGFLAGS += -DBACKTRACE_ENABLE=0
|
||||
endif
|
||||
ifeq (,$(filter Windows%, $(OS)))
|
||||
ifeq ($(BACKTRACE), 1)
|
||||
DEBUGFLAGS += -DBACKTRACE_ENABLE=1
|
||||
DEBUGFLAGS_LD += -rdynamic
|
||||
endif
|
||||
endif
|
||||
|
||||
SET_CACHE_DIRECTORY = \
|
||||
+$(MAKE) --no-print-directory $@ \
|
||||
BUILD_DIR=obj/$(HASH_DIR) \
|
||||
CPPFLAGS="$(CPPFLAGS)" \
|
||||
CFLAGS="$(CFLAGS)" \
|
||||
LDFLAGS="$(LDFLAGS)" \
|
||||
LDLIBS="$(LDLIBS)" \
|
||||
ZSTD_ALL_SRC="$(ZSTD_ALL_SRC)"
|
||||
|
||||
|
||||
.PHONY: all
|
||||
all: zstd zstd-compress zstd-decompress zstd-small
|
||||
|
||||
.PHONY: allVariants
|
||||
allVariants: all zstd-frugal zstd-nolegacy zstd-dictBuilder
|
||||
|
||||
.PHONY: zstd # must always be run
|
||||
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
|
||||
zstd : LDFLAGS += $(THREAD_LD) $(DEBUGFLAGS_LD)
|
||||
zstd : LDLIBS += $(ZLIBLD) $(LZMALD) $(LZ4LD)
|
||||
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
zstd : $(RES_FILE)
|
||||
endif
|
||||
|
||||
ifndef BUILD_DIR
|
||||
# generate BUILD_DIR from flags
|
||||
|
||||
zstd:
|
||||
$(SET_CACHE_DIRECTORY)
|
||||
|
||||
else
|
||||
# BUILD_DIR is defined
|
||||
|
||||
ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ))
|
||||
$(BUILD_DIR)/zstd : $(ZSTD_OBJ)
|
||||
@echo "$(THREAD_MSG)"
|
||||
@echo "$(ZLIB_MSG)"
|
||||
@echo "$(LZMA_MSG)"
|
||||
@echo "$(LZ4_MSG)"
|
||||
@echo LINK $@
|
||||
$(CC) $(FLAGS) $^ $(LDLIBS) -o $@$(EXT)
|
||||
|
||||
ifeq ($(HAVE_HASH),1)
|
||||
SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd$(EXT) 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
|
||||
DSTBIN_HASH = $(shell cat zstd$(EXT) 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
|
||||
BIN_ISDIFFERENT = $(if $(filter $(SRCBIN_HASH),$(DSTBIN_HASH)),0,1)
|
||||
else
|
||||
BIN_ISDIFFERENT = 1
|
||||
endif
|
||||
|
||||
zstd : $(BUILD_DIR)/zstd
|
||||
if [ $(BIN_ISDIFFERENT) -eq 1 ]; then \
|
||||
cp -f $<$(EXT) $@$(EXT); \
|
||||
echo zstd build completed; \
|
||||
else \
|
||||
echo zstd already built; \
|
||||
fi
|
||||
|
||||
endif # BUILD_DIR
|
||||
|
||||
|
||||
CLEAN += zstd
|
||||
.PHONY: zstd-release
|
||||
zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
|
||||
zstd-release: DEBUGFLAGS_LD :=
|
||||
zstd-release: zstd
|
||||
|
||||
CLEAN += zstd32
|
||||
zstd32 : CPPFLAGS += $(THREAD_CPP)
|
||||
zstd32 : LDFLAGS += $(THREAD_LD)
|
||||
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
zstd32 : $(RES32_FILE)
|
||||
endif
|
||||
zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
## zstd-nolegacy: same scope as zstd, with removed support of legacy formats
|
||||
CLEAN += zstd-nolegacy
|
||||
zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
|
||||
zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
|
||||
zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
|
||||
|
||||
.PHONY: zstd-nomt
|
||||
zstd-nomt : THREAD_CPP :=
|
||||
zstd-nomt : THREAD_LD :=
|
||||
zstd-nomt : THREAD_MSG := - multi-threading disabled
|
||||
zstd-nomt : zstd
|
||||
|
||||
.PHONY: zstd-nogz
|
||||
zstd-nogz : ZLIBCPP :=
|
||||
zstd-nogz : ZLIBLD :=
|
||||
zstd-nogz : ZLIB_MSG := - gzip support is disabled
|
||||
zstd-nogz : zstd
|
||||
|
||||
.PHONY: zstd-noxz
|
||||
zstd-noxz : LZMACPP :=
|
||||
zstd-noxz : LZMALD :=
|
||||
zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
|
||||
zstd-noxz : zstd
|
||||
|
||||
## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version)
|
||||
.PHONY: zstd-dll
|
||||
zstd-dll : LDFLAGS+= -L$(LIB_BINDIR)
|
||||
zstd-dll : LDLIBS += -lzstd
|
||||
zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c pool.c threading.c
|
||||
zstd-dll : zstd
|
||||
|
||||
|
||||
## zstd-pgo: zstd executable optimized with PGO.
|
||||
.PHONY: zstd-pgo
|
||||
zstd-pgo : LLVM_PROFDATA?=llvm-profdata
|
||||
zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
|
||||
zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Wno-error=missing-profile -Wno-error=coverage-mismatch)
|
||||
zstd-pgo :
|
||||
$(MAKE) clean HASH_DIR=$(HASH_DIR)
|
||||
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
|
||||
./zstd -b19i1 $(PROFILE_WITH)
|
||||
./zstd -b16i1 $(PROFILE_WITH)
|
||||
./zstd -b9i2 $(PROFILE_WITH)
|
||||
./zstd -b $(PROFILE_WITH)
|
||||
./zstd -b7i2 $(PROFILE_WITH)
|
||||
./zstd -b5 $(PROFILE_WITH)
|
||||
ifndef BUILD_DIR
|
||||
$(RM) zstd obj/$(HASH_DIR)/zstd obj/$(HASH_DIR)/*.o
|
||||
else
|
||||
$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
|
||||
endif
|
||||
case $(CC) in *clang*) if ! [ -e default.profdata ]; then $(LLVM_PROFDATA) merge -output=default.profdata default*.profraw; fi ;; esac
|
||||
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_USE_FLAGS)"
|
||||
|
||||
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
|
||||
CLEAN += zstd-small zstd-frugal
|
||||
zstd-small: CFLAGS = -Os -Wl,-s
|
||||
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
CLEAN += zstd-decompress
|
||||
zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
CLEAN += zstd-compress
|
||||
zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
|
||||
CLEAN += zstd-dictBuilder
|
||||
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
|
||||
|
||||
CLEAN += zstdmt
|
||||
zstdmt: zstd
|
||||
ln -sf zstd zstdmt
|
||||
|
||||
.PHONY: generate_res
|
||||
generate_res: $(RES64_FILE) $(RES32_FILE)
|
||||
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
RC ?= windres
|
||||
# https://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
|
||||
$(RES64_FILE): windres/zstd.rc
|
||||
$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
|
||||
$(RES32_FILE): windres/zstd.rc
|
||||
$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-i386
|
||||
endif
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) $(CLEAN) core *.o tmp* result* dictionary *.zst \
|
||||
*.gcda default*.profraw default.profdata have_zlib
|
||||
$(RM) -r obj/*
|
||||
@echo Cleaning completed
|
||||
|
||||
MD2ROFF = ronn
|
||||
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
|
||||
|
||||
zstd.1: zstd.1.md ../lib/zstd.h
|
||||
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||
|
||||
zstdgrep.1: zstdgrep.1.md ../lib/zstd.h
|
||||
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||
|
||||
zstdless.1: zstdless.1.md ../lib/zstd.h
|
||||
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
|
||||
|
||||
.PHONY: man
|
||||
man: zstd.1 zstdgrep.1 zstdless.1
|
||||
|
||||
.PHONY: clean-man
|
||||
clean-man:
|
||||
$(RM) zstd.1
|
||||
$(RM) zstdgrep.1
|
||||
$(RM) zstdless.1
|
||||
|
||||
.PHONY: preview-man
|
||||
preview-man: clean-man man
|
||||
man ./zstd.1
|
||||
man ./zstdgrep.1
|
||||
man ./zstdless.1
|
||||
|
||||
|
||||
# Generate .h dependencies automatically
|
||||
|
||||
DEPFLAGS = -MT $@ -MMD -MP -MF
|
||||
|
||||
$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR)
|
||||
@echo CC $@
|
||||
$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $<
|
||||
|
||||
$(BUILD_DIR)/%.o : %.S | $(BUILD_DIR)
|
||||
@echo AS $@
|
||||
$(COMPILE.S) $(OUTPUT_OPTION) $<
|
||||
|
||||
MKDIR ?= mkdir
|
||||
$(BUILD_DIR): ; $(MKDIR) -p $@
|
||||
|
||||
DEPFILES := $(ZSTD_OBJ:.o=.d)
|
||||
$(DEPFILES):
|
||||
|
||||
include $(wildcard $(DEPFILES))
|
||||
|
||||
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT%,$(UNAME)))
|
||||
|
||||
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
|
||||
EGREP_OPTIONS ?=
|
||||
ifeq ($(HAVE_COLORNEVER), 1)
|
||||
EGREP_OPTIONS += --color=never
|
||||
endif
|
||||
EGREP = egrep $(EGREP_OPTIONS)
|
||||
AWK = awk
|
||||
|
||||
# Print a two column output of targets and their description. To add a target description, put a
|
||||
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
|
||||
#
|
||||
## list: Print all targets and their descriptions (if provided)
|
||||
.PHONY: list
|
||||
list:
|
||||
TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
|
||||
| $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
|
||||
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
|
||||
{ \
|
||||
printf "Target Name\tDescription\n"; \
|
||||
printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
|
||||
for target in $$TARGETS; do \
|
||||
line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
|
||||
description=$$(echo $$line | $(AWK) '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
|
||||
printf "$$target\t$$description\n"; \
|
||||
done \
|
||||
} | column -t -s $$'\t'
|
||||
|
||||
|
||||
DESTDIR ?=
|
||||
# directory variables : GNU conventions prefer lowercase
|
||||
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
|
||||
# support both lower and uppercase (BSD), use uppercase in script
|
||||
prefix ?= /usr/local
|
||||
PREFIX ?= $(prefix)
|
||||
exec_prefix ?= $(PREFIX)
|
||||
bindir ?= $(exec_prefix)/bin
|
||||
BINDIR ?= $(bindir)
|
||||
datarootdir ?= $(PREFIX)/share
|
||||
mandir ?= $(datarootdir)/man
|
||||
man1dir ?= $(mandir)/man1
|
||||
|
||||
ifneq (,$(filter OpenBSD NetBSD DragonFly SunOS,$(UNAME)))
|
||||
MANDIR ?= $(PREFIX)/man
|
||||
MAN1DIR ?= $(MANDIR)/man1
|
||||
else
|
||||
MAN1DIR ?= $(man1dir)
|
||||
endif
|
||||
|
||||
ifneq (,$(filter SunOS,$(UNAME)))
|
||||
INSTALL ?= ginstall
|
||||
else
|
||||
INSTALL ?= install
|
||||
endif
|
||||
|
||||
INSTALL_PROGRAM ?= $(INSTALL)
|
||||
INSTALL_SCRIPT ?= $(INSTALL_PROGRAM)
|
||||
INSTALL_DATA ?= $(INSTALL) -m 644
|
||||
INSTALL_MAN ?= $(INSTALL_DATA)
|
||||
|
||||
.PHONY: install
|
||||
install:
|
||||
# generate zstd only if not already present
|
||||
[ -e zstd ] || $(MAKE) zstd-release
|
||||
[ -e $(DESTDIR)$(BINDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
|
||||
[ -e $(DESTDIR)$(MAN1DIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(MAN1DIR)/
|
||||
@echo Installing binaries
|
||||
$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
|
||||
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
|
||||
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
|
||||
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
|
||||
$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
|
||||
$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
|
||||
@echo Installing man pages
|
||||
$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
|
||||
ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
|
||||
ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
|
||||
$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
|
||||
$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
|
||||
@echo zstd installation completed
|
||||
|
||||
.PHONY: uninstall
|
||||
uninstall:
|
||||
$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
|
||||
$(RM) $(DESTDIR)$(BINDIR)/zstdless
|
||||
$(RM) $(DESTDIR)$(BINDIR)/zstdcat
|
||||
$(RM) $(DESTDIR)$(BINDIR)/unzstd
|
||||
$(RM) $(DESTDIR)$(BINDIR)/zstdmt
|
||||
$(RM) $(DESTDIR)$(BINDIR)/zstd
|
||||
$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
|
||||
$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
|
||||
$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
|
||||
$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
|
||||
$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
|
||||
@echo zstd programs successfully uninstalled
|
||||
|
||||
endif
|
||||
@@ -1,22 +0,0 @@
|
||||
cxx_library(
|
||||
name='zlib_wrapper',
|
||||
visibility=['PUBLIC'],
|
||||
exported_linker_flags=['-lz'],
|
||||
header_namespace='',
|
||||
exported_headers=['zstd_zlibwrapper.h'],
|
||||
headers=[
|
||||
'gzcompatibility.h',
|
||||
'gzguts.h',
|
||||
],
|
||||
srcs=glob(['*.c']),
|
||||
deps=[
|
||||
'//lib:zstd',
|
||||
'//lib:zstd_common',
|
||||
],
|
||||
)
|
||||
|
||||
cxx_binary(
|
||||
name='minigzip',
|
||||
srcs=['examples/minigzip.c'],
|
||||
deps=[':zlib_wrapper'],
|
||||
)
|
||||
@@ -1,120 +0,0 @@
|
||||
# Makefile for example of using zstd wrapper for zlib
|
||||
#
|
||||
# make - compiles examples
|
||||
# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
|
||||
# make test - runs examples
|
||||
|
||||
|
||||
# Paths to static and dynamic zlib and zstd libraries
|
||||
# Use "make ZLIB_PATH=path/to/zlib ZLIB_LIBRARY=path/to/libz.so" to select a path to library
|
||||
ZLIB_LIBRARY ?= -lz
|
||||
ZLIB_PATH ?= .
|
||||
|
||||
ZSTDLIBDIR = ../lib
|
||||
ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
|
||||
ZLIBWRAPPER_PATH = .
|
||||
GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
|
||||
EXAMPLE_PATH = examples
|
||||
PROGRAMS_PATH = ../programs
|
||||
TEST_FILE = ../doc/zstd_compression_format.md
|
||||
|
||||
vpath %.c $(PROGRAMS_PATH) $(EXAMPLE_PATH) $(ZLIBWRAPPER_PATH)
|
||||
|
||||
|
||||
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \
|
||||
-I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
|
||||
STDFLAGS = -std=c89 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \
|
||||
-DNO_snprintf -DNO_vsnprintf # strict ANSI C89 is missing these prototypes
|
||||
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
|
||||
-Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
|
||||
-Wstrict-aliasing=1
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += $(STDFLAGS) $(DEBUGFLAGS)
|
||||
CPPFLAGS += $(MOREFLAGS)
|
||||
LDLIBS += $(ZLIB_LIBRARY)
|
||||
|
||||
# Define *.exe as extension for Windows systems
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
EXT =.exe
|
||||
else
|
||||
EXT =
|
||||
endif
|
||||
|
||||
default : release
|
||||
|
||||
release : STDFLAGS =
|
||||
release : DEBUGFLAGS =
|
||||
release : all
|
||||
|
||||
all: fitblk example zwrapbench minigzip
|
||||
|
||||
test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd
|
||||
./example
|
||||
./example_zstd
|
||||
./fitblk 10240 <$(TEST_FILE)
|
||||
./fitblk 40960 <$(TEST_FILE)
|
||||
./fitblk_zstd 10240 <$(TEST_FILE)
|
||||
./fitblk_zstd 40960 <$(TEST_FILE)
|
||||
@echo ---- minigzip start ----
|
||||
./minigzip_zstd example$(EXT)
|
||||
#cp example$(EXT).gz example$(EXT)_zstd.gz
|
||||
./minigzip_zstd -d example$(EXT).gz
|
||||
./minigzip example$(EXT)
|
||||
#cp example$(EXT).gz example$(EXT)_gz.gz
|
||||
./minigzip_zstd -d example$(EXT).gz
|
||||
@echo ---- minigzip end ----
|
||||
./zwrapbench -qi1b3B1K $(TEST_FILE)
|
||||
./zwrapbench -rqi1b1e3 ../lib
|
||||
|
||||
.PHONY: test-valgrind
|
||||
#test-valgrind: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
|
||||
test-valgrind: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
|
||||
test-valgrind: clean example fitblk example_zstd fitblk_zstd zwrapbench
|
||||
@echo "\n ---- valgrind tests ----"
|
||||
$(VALGRIND) ./example
|
||||
$(VALGRIND) ./example_zstd
|
||||
$(VALGRIND) ./fitblk 10240 <$(TEST_FILE)
|
||||
$(VALGRIND) ./fitblk 40960 <$(TEST_FILE)
|
||||
$(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE)
|
||||
$(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE)
|
||||
$(VALGRIND) ./zwrapbench -qi1b3B1K $(TEST_FILE)
|
||||
$(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests
|
||||
|
||||
#.c.o:
|
||||
# $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
|
||||
|
||||
minigzip: minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
|
||||
minigzip_zstd: minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
|
||||
|
||||
example: example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
|
||||
example_zstd: example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY)
|
||||
$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
|
||||
|
||||
fitblk: fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
|
||||
fitblk_zstd: fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
|
||||
$(LINK.o) $^ $(LDLIBS) $(OUTPUT_OPTION)
|
||||
|
||||
zwrapbench: zwrapbench.o zstd_zlibwrapper.o util.o timefn.o datagen.o $(ZSTDLIBRARY)
|
||||
|
||||
|
||||
zstd_zlibwrapper.o: zstd_zlibwrapper.h
|
||||
|
||||
zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
|
||||
zstdTurnedOn_zlibwrapper.o: zstd_zlibwrapper.c zstd_zlibwrapper.h
|
||||
$(COMPILE.c) $< $(OUTPUT_OPTION)
|
||||
|
||||
|
||||
$(ZSTDLIBRARY):
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
|
||||
|
||||
$(ZSTDLIBDIR)/libzstd.so:
|
||||
$(MAKE) -C $(ZSTDLIBDIR) libzstd
|
||||
|
||||
|
||||
clean:
|
||||
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o foo.gz example$(EXT) example_zstd$(EXT) fitblk$(EXT) fitblk_zstd$(EXT) zwrapbench$(EXT) minigzip$(EXT) minigzip_zstd$(EXT)
|
||||
@echo Cleaning completed
|
||||
@@ -1,598 +0,0 @@
|
||||
/* example.c contains minimal changes required to be compiled with zlibWrapper:
|
||||
* - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h"
|
||||
* - test_flush() and test_sync() use functions not supported by zlibWrapper
|
||||
therefore they are disabled while zstd compression is turned on */
|
||||
|
||||
/* example.c -- usage example of the zlib compression library
|
||||
*/
|
||||
/*
|
||||
Copyright (c) 1995-2006, 2011 Jean-loup Gailly
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any damages
|
||||
arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not
|
||||
claim that you wrote the original software. If you use this software
|
||||
in a product, an acknowledgement in the product documentation would be
|
||||
appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
|
||||
/* @(#) $Id$ */
|
||||
|
||||
#include "zstd_zlibwrapper.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef STDC
|
||||
# include <string.h>
|
||||
# include <stdlib.h>
|
||||
#endif
|
||||
|
||||
#if defined(VMS) || defined(RISCOS)
|
||||
# define TESTFILE "foo-gz"
|
||||
#else
|
||||
# define TESTFILE "foo.gz"
|
||||
#endif
|
||||
|
||||
#define CHECK_ERR(err, msg) { \
|
||||
if (err != Z_OK) { \
|
||||
fprintf(stderr, "%s error: %d\n", msg, err); \
|
||||
exit(1); \
|
||||
} \
|
||||
}
|
||||
|
||||
z_const char hello[] = "hello, hello! I said hello, hello!";
|
||||
/* "hello world" would be more standard, but the repeated "hello"
|
||||
* stresses the compression code better, sorry...
|
||||
*/
|
||||
|
||||
const char dictionary[] = "hello, hello!";
|
||||
uLong dictId; /* Adler32 value of the dictionary */
|
||||
|
||||
void test_deflate _Z_OF((Byte *compr, uLong comprLen));
|
||||
void test_inflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_large_deflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_large_inflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_flush _Z_OF((Byte *compr, uLong *comprLen));
|
||||
void test_sync _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_dict_deflate _Z_OF((Byte *compr, uLong comprLen));
|
||||
void test_dict_inflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
int main _Z_OF((int argc, char *argv[]));
|
||||
|
||||
|
||||
#ifdef Z_SOLO
|
||||
|
||||
void *myalloc _Z_OF((void *, unsigned, unsigned));
|
||||
void myfree _Z_OF((void *, void *));
|
||||
|
||||
void *myalloc(void *q, unsigned n, unsigned m)
|
||||
{
|
||||
void *buf = calloc(n, m);
|
||||
q = Z_NULL;
|
||||
/* printf("myalloc %p n=%d m=%d\n", buf, n, m); */
|
||||
return buf;
|
||||
}
|
||||
|
||||
void myfree(void *q, void *p)
|
||||
{
|
||||
/* printf("myfree %p\n", p); */
|
||||
q = Z_NULL;
|
||||
free(p);
|
||||
}
|
||||
|
||||
static alloc_func zalloc = myalloc;
|
||||
static free_func zfree = myfree;
|
||||
|
||||
#else /* !Z_SOLO */
|
||||
|
||||
static alloc_func zalloc = (alloc_func)0;
|
||||
static free_func zfree = (free_func)0;
|
||||
|
||||
void test_compress _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_gzio _Z_OF((const char *fname,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
|
||||
/* ===========================================================================
|
||||
* Test compress() and uncompress()
|
||||
*/
|
||||
void test_compress(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen) {
|
||||
int err;
|
||||
uLong len = (uLong)strlen(hello)+1;
|
||||
|
||||
err = compress(compr, &comprLen, (const Bytef*)hello, len);
|
||||
CHECK_ERR(err, "compress");
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
err = uncompress(uncompr, &uncomprLen, compr, comprLen);
|
||||
CHECK_ERR(err, "uncompress");
|
||||
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad uncompress\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("uncompress(): %s\n", (char *)uncompr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test read/write of .gz files
|
||||
*/
|
||||
void test_gzio(const char *fname, Byte *uncompr, uLong uncomprLen) {
|
||||
#ifdef NO_GZCOMPRESS
|
||||
fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n");
|
||||
#else
|
||||
int err;
|
||||
int len = (int)strlen(hello)+1;
|
||||
gzFile file;
|
||||
z_off_t pos;
|
||||
|
||||
file = gzopen(fname, "wb");
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "gzopen error\n");
|
||||
exit(1);
|
||||
}
|
||||
gzputc(file, 'h');
|
||||
if (gzputs(file, "ello") != 4) {
|
||||
fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
if (gzprintf(file, ", %s! I said hello, hello!", "hello") != 8+21) {
|
||||
fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
gzseek(file, 1L, SEEK_CUR); /* add one zero byte */
|
||||
gzclose(file);
|
||||
|
||||
file = gzopen(fname, "rb");
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "gzopen error\n");
|
||||
exit(1);
|
||||
}
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
if (gzread(file, uncompr, (unsigned)uncomprLen) != len) {
|
||||
fprintf(stderr, "gzread err: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad gzread: %s\n", (char*)uncompr);
|
||||
exit(1);
|
||||
} else {
|
||||
printf("gzread(): %s\n", (char*)uncompr);
|
||||
}
|
||||
|
||||
pos = gzseek(file, -8L, SEEK_CUR);
|
||||
if (pos != 6+21 || gztell(file) != pos) {
|
||||
fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n",
|
||||
(long)pos, (long)gztell(file));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (gzgetc(file) != ' ') {
|
||||
fprintf(stderr, "gzgetc error\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (gzungetc(' ', file) != ' ') {
|
||||
fprintf(stderr, "gzungetc error\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
gzgets(file, (char*)uncompr, (int)uncomprLen);
|
||||
if (strlen((char*)uncompr) != 7) { /* " hello!" */
|
||||
fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
if (strcmp((char*)uncompr, hello + 6+21)) {
|
||||
fprintf(stderr, "bad gzgets after gzseek\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("gzgets() after gzseek: %s\n", (char*)uncompr);
|
||||
}
|
||||
|
||||
gzclose(file);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* Z_SOLO */
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with small buffers
|
||||
*/
|
||||
void test_deflate(Byte *compr, uLong comprLen) {
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
uLong len = (uLong)strlen(hello)+1;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
c_stream.next_in = (z_const unsigned char *)hello;
|
||||
c_stream.next_out = compr;
|
||||
|
||||
while (c_stream.total_in != len && c_stream.total_out < comprLen) {
|
||||
c_stream.avail_in = c_stream.avail_out = 1; /* force small buffers */
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
}
|
||||
/* Finish the stream, still forcing small buffers: */
|
||||
for (;;) {
|
||||
c_stream.avail_out = 1;
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
CHECK_ERR(err, "deflate");
|
||||
}
|
||||
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflate() with small buffers
|
||||
*/
|
||||
void test_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen) {
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = 0;
|
||||
d_stream.next_out = uncompr;
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
while (d_stream.total_out < uncomprLen && d_stream.total_in < comprLen) {
|
||||
d_stream.avail_in = d_stream.avail_out = 1; /* force small buffers */
|
||||
err = inflate(&d_stream, Z_NO_FLUSH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
CHECK_ERR(err, "inflate");
|
||||
}
|
||||
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad inflate\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("inflate(): %s\n", (char *)uncompr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with large buffers and dynamic change of compression level
|
||||
*/
|
||||
void test_large_deflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen) {
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_BEST_SPEED);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
c_stream.next_out = compr;
|
||||
c_stream.avail_out = (uInt)comprLen;
|
||||
|
||||
/* At this point, uncompr is still mostly zeroes, so it should compress
|
||||
* very well:
|
||||
*/
|
||||
c_stream.next_in = uncompr;
|
||||
c_stream.avail_in = (uInt)uncomprLen;
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
if (c_stream.avail_in != 0) {
|
||||
fprintf(stderr, "deflate not greedy\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* Feed in already compressed data and switch to no compression: */
|
||||
deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY);
|
||||
c_stream.next_in = compr;
|
||||
c_stream.avail_in = (uInt)comprLen/2;
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
|
||||
/* Switch back to compressing mode: */
|
||||
deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED);
|
||||
c_stream.next_in = uncompr;
|
||||
c_stream.avail_in = (uInt)uncomprLen;
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
fprintf(stderr, "deflate should report Z_STREAM_END\n");
|
||||
exit(1);
|
||||
}
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflate() with large buffers
|
||||
*/
|
||||
void test_large_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen) {
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = (uInt)comprLen;
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
for (;;) {
|
||||
d_stream.next_out = uncompr; /* discard the output */
|
||||
d_stream.avail_out = (uInt)uncomprLen;
|
||||
err = inflate(&d_stream, Z_NO_FLUSH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
CHECK_ERR(err, "large inflate");
|
||||
}
|
||||
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
if (d_stream.total_out != 2*uncomprLen + comprLen/2) {
|
||||
fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out);
|
||||
exit(1);
|
||||
} else {
|
||||
printf("large_inflate(): OK\n");
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with full flush
|
||||
*/
|
||||
void test_flush(Byte *compr, uLong *comprLen) {
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
uInt len = (uInt)strlen(hello)+1;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
c_stream.next_in = (z_const unsigned char *)hello;
|
||||
c_stream.next_out = compr;
|
||||
c_stream.avail_in = 3;
|
||||
c_stream.avail_out = (uInt)*comprLen;
|
||||
err = deflate(&c_stream, Z_FULL_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
|
||||
compr[3]++; /* force an error in first compressed block */
|
||||
c_stream.avail_in = len - 3;
|
||||
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
CHECK_ERR(err, "deflate");
|
||||
}
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
|
||||
*comprLen = c_stream.total_out;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflateSync()
|
||||
*/
|
||||
void test_sync(Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen) {
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = 2; /* just read the zlib header */
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
d_stream.next_out = uncompr;
|
||||
d_stream.avail_out = (uInt)uncomprLen;
|
||||
|
||||
inflate(&d_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "inflate");
|
||||
|
||||
d_stream.avail_in = (uInt)comprLen-2; /* read all compressed data */
|
||||
err = inflateSync(&d_stream); /* but skip the damaged part */
|
||||
CHECK_ERR(err, "inflateSync");
|
||||
|
||||
err = inflate(&d_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
fprintf(stderr, "inflate reported %i != %i (Z_STREAM_END)\n", err, Z_STREAM_END);
|
||||
/* Because of incorrect adler32 */
|
||||
exit(1);
|
||||
}
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
printf("after inflateSync(): hel%s\n", (char *)uncompr);
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with preset dictionary
|
||||
*/
|
||||
void test_dict_deflate(Byte *compr, uLong comprLen) {
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_BEST_COMPRESSION);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
err = deflateSetDictionary(&c_stream,
|
||||
(const Bytef*)dictionary, (int)sizeof(dictionary));
|
||||
CHECK_ERR(err, "deflateSetDictionary");
|
||||
|
||||
dictId = c_stream.adler;
|
||||
c_stream.next_out = compr;
|
||||
c_stream.avail_out = (uInt)comprLen;
|
||||
|
||||
c_stream.next_in = (z_const unsigned char *)hello;
|
||||
c_stream.avail_in = (uInt)strlen(hello)+1;
|
||||
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
fprintf(stderr, "deflate should report Z_STREAM_END\n");
|
||||
exit(1);
|
||||
}
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflate() with a preset dictionary
|
||||
*/
|
||||
void test_dict_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen) {
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = (uInt)comprLen;
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
d_stream.next_out = uncompr;
|
||||
d_stream.avail_out = (uInt)uncomprLen;
|
||||
|
||||
for (;;) {
|
||||
err = inflate(&d_stream, Z_NO_FLUSH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
if (err == Z_NEED_DICT) {
|
||||
if (d_stream.adler != dictId) {
|
||||
fprintf(stderr, "unexpected dictionary");
|
||||
exit(1);
|
||||
}
|
||||
err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary,
|
||||
(int)sizeof(dictionary));
|
||||
}
|
||||
CHECK_ERR(err, "inflate with dict");
|
||||
}
|
||||
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad inflate with dict\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("inflate with dictionary: %s\n", (char *)uncompr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Usage: example [output.gz [input.gz]]
|
||||
*/
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
Byte *compr, *uncompr;
|
||||
uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */
|
||||
uLong uncomprLen = comprLen;
|
||||
static const char* myVersion = ZLIB_VERSION;
|
||||
|
||||
if (zlibVersion()[0] != myVersion[0]) {
|
||||
fprintf(stderr, "incompatible zlib version\n");
|
||||
exit(1);
|
||||
|
||||
} else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) {
|
||||
fprintf(stderr, "warning: different zlib version\n");
|
||||
}
|
||||
|
||||
printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n",
|
||||
ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags());
|
||||
if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
|
||||
|
||||
compr = (Byte*)calloc((uInt)comprLen, 1);
|
||||
uncompr = (Byte*)calloc((uInt)uncomprLen, 1);
|
||||
/* compr and uncompr are cleared to avoid reading uninitialized
|
||||
* data and to ensure that uncompr compresses well.
|
||||
*/
|
||||
if (compr == Z_NULL || uncompr == Z_NULL) {
|
||||
printf("out of memory\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifdef Z_SOLO
|
||||
argc = strlen(argv[0]);
|
||||
#else
|
||||
test_compress(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
test_gzio((argc > 1 ? argv[1] : TESTFILE),
|
||||
uncompr, uncomprLen);
|
||||
#endif
|
||||
|
||||
test_deflate(compr, comprLen);
|
||||
test_inflate(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
test_large_deflate(compr, comprLen, uncompr, uncomprLen);
|
||||
test_large_inflate(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
if (!ZWRAP_isUsingZSTDcompression()) {
|
||||
test_flush(compr, &comprLen);
|
||||
test_sync(compr, comprLen, uncompr, uncomprLen);
|
||||
}
|
||||
comprLen = uncomprLen;
|
||||
|
||||
test_dict_deflate(compr, comprLen);
|
||||
test_dict_inflate(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
free(compr);
|
||||
free(uncompr);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,599 +0,0 @@
|
||||
/* example.c -- usage example of the zlib compression library
|
||||
*/
|
||||
/*
|
||||
Copyright (c) 1995-2006, 2011 Jean-loup Gailly
|
||||
|
||||
This software is provided 'as-is', without any express or implied
|
||||
warranty. In no event will the authors be held liable for any damages
|
||||
arising from the use of this software.
|
||||
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
|
||||
1. The origin of this software must not be misrepresented; you must not
|
||||
claim that you wrote the original software. If you use this software
|
||||
in a product, an acknowledgement in the product documentation would be
|
||||
appreciated but is not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
*/
|
||||
|
||||
/* @(#) $Id$ */
|
||||
|
||||
#include "zlib.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef STDC
|
||||
# include <string.h>
|
||||
# include <stdlib.h>
|
||||
#endif
|
||||
|
||||
#if defined(VMS) || defined(RISCOS)
|
||||
# define TESTFILE "foo-gz"
|
||||
#else
|
||||
# define TESTFILE "foo.gz"
|
||||
#endif
|
||||
|
||||
#define CHECK_ERR(err, msg) { \
|
||||
if (err != Z_OK) { \
|
||||
fprintf(stderr, "%s error: %d\n", msg, err); \
|
||||
exit(1); \
|
||||
} \
|
||||
}
|
||||
|
||||
z_const char hello[] = "hello, hello!";
|
||||
/* "hello world" would be more standard, but the repeated "hello"
|
||||
* stresses the compression code better, sorry...
|
||||
*/
|
||||
|
||||
const char dictionary[] = "hello";
|
||||
uLong dictId; /* Adler32 value of the dictionary */
|
||||
|
||||
void test_deflate _Z_OF((Byte *compr, uLong comprLen));
|
||||
void test_inflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_large_deflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_large_inflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_flush _Z_OF((Byte *compr, uLong *comprLen));
|
||||
void test_sync _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_dict_deflate _Z_OF((Byte *compr, uLong comprLen));
|
||||
void test_dict_inflate _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
int main _Z_OF((int argc, char *argv[]));
|
||||
|
||||
|
||||
#ifdef Z_SOLO
|
||||
|
||||
void *myalloc _Z_OF((void *, unsigned, unsigned));
|
||||
void myfree _Z_OF((void *, void *));
|
||||
|
||||
void *myalloc(q, n, m)
|
||||
void *q;
|
||||
unsigned n, m;
|
||||
{
|
||||
q = Z_NULL;
|
||||
return calloc(n, m);
|
||||
}
|
||||
|
||||
void myfree(void *q, void *p)
|
||||
{
|
||||
q = Z_NULL;
|
||||
free(p);
|
||||
}
|
||||
|
||||
static alloc_func zalloc = myalloc;
|
||||
static free_func zfree = myfree;
|
||||
|
||||
#else /* !Z_SOLO */
|
||||
|
||||
static alloc_func zalloc = (alloc_func)0;
|
||||
static free_func zfree = (free_func)0;
|
||||
|
||||
void test_compress _Z_OF((Byte *compr, uLong comprLen,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
void test_gzio _Z_OF((const char *fname,
|
||||
Byte *uncompr, uLong uncomprLen));
|
||||
|
||||
/* ===========================================================================
|
||||
* Test compress() and uncompress()
|
||||
*/
|
||||
void test_compress(Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen)
|
||||
{
|
||||
int err;
|
||||
uLong len = (uLong)strlen(hello)+1;
|
||||
|
||||
err = compress(compr, &comprLen, (const Bytef*)hello, len);
|
||||
CHECK_ERR(err, "compress");
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
err = uncompress(uncompr, &uncomprLen, compr, comprLen);
|
||||
CHECK_ERR(err, "uncompress");
|
||||
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad uncompress\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("uncompress(): %s\n", (char *)uncompr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test read/write of .gz files
|
||||
*/
|
||||
void test_gzio(const char *fname /* compressed file name */, Byte *uncompr,
|
||||
uLong uncomprLen)
|
||||
{
|
||||
#ifdef NO_GZCOMPRESS
|
||||
fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n");
|
||||
#else
|
||||
int err;
|
||||
int len = (int)strlen(hello)+1;
|
||||
gzFile file;
|
||||
z_off_t pos;
|
||||
|
||||
file = gzopen(fname, "wb");
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "gzopen error\n");
|
||||
exit(1);
|
||||
}
|
||||
gzputc(file, 'h');
|
||||
if (gzputs(file, "ello") != 4) {
|
||||
fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
if (gzprintf(file, ", %s!", "hello") != 8) {
|
||||
fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
gzseek(file, 1L, SEEK_CUR); /* add one zero byte */
|
||||
gzclose(file);
|
||||
|
||||
file = gzopen(fname, "rb");
|
||||
if (file == NULL) {
|
||||
fprintf(stderr, "gzopen error\n");
|
||||
exit(1);
|
||||
}
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
if (gzread(file, uncompr, (unsigned)uncomprLen) != len) {
|
||||
fprintf(stderr, "gzread err: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad gzread: %s\n", (char*)uncompr);
|
||||
exit(1);
|
||||
} else {
|
||||
printf("gzread(): %s\n", (char*)uncompr);
|
||||
}
|
||||
|
||||
pos = gzseek(file, -8L, SEEK_CUR);
|
||||
if (pos != 6 || gztell(file) != pos) {
|
||||
fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n",
|
||||
(long)pos, (long)gztell(file));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (gzgetc(file) != ' ') {
|
||||
fprintf(stderr, "gzgetc error\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (gzungetc(' ', file) != ' ') {
|
||||
fprintf(stderr, "gzungetc error\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
gzgets(file, (char*)uncompr, (int)uncomprLen);
|
||||
if (strlen((char*)uncompr) != 7) { /* " hello!" */
|
||||
fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err));
|
||||
exit(1);
|
||||
}
|
||||
if (strcmp((char*)uncompr, hello + 6)) {
|
||||
fprintf(stderr, "bad gzgets after gzseek\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("gzgets() after gzseek: %s\n", (char*)uncompr);
|
||||
}
|
||||
|
||||
gzclose(file);
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* Z_SOLO */
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with small buffers
|
||||
*/
|
||||
void test_deflate(Byte *compr, uLong comprLen)
|
||||
{
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
uLong len = (uLong)strlen(hello)+1;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
c_stream.next_in = (z_const unsigned char *)hello;
|
||||
c_stream.next_out = compr;
|
||||
|
||||
while (c_stream.total_in != len && c_stream.total_out < comprLen) {
|
||||
c_stream.avail_in = c_stream.avail_out = 1; /* force small buffers */
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
}
|
||||
/* Finish the stream, still forcing small buffers: */
|
||||
for (;;) {
|
||||
c_stream.avail_out = 1;
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
CHECK_ERR(err, "deflate");
|
||||
}
|
||||
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflate() with small buffers
|
||||
*/
|
||||
void test_inflate(Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen)
|
||||
{
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = 0;
|
||||
d_stream.next_out = uncompr;
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
while (d_stream.total_out < uncomprLen && d_stream.total_in < comprLen) {
|
||||
d_stream.avail_in = d_stream.avail_out = 1; /* force small buffers */
|
||||
err = inflate(&d_stream, Z_NO_FLUSH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
CHECK_ERR(err, "inflate");
|
||||
}
|
||||
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad inflate\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("inflate(): %s\n", (char *)uncompr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with large buffers and dynamic change of compression level
|
||||
*/
|
||||
void test_large_deflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen)
|
||||
{
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_BEST_SPEED);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
c_stream.next_out = compr;
|
||||
c_stream.avail_out = (uInt)comprLen;
|
||||
|
||||
/* At this point, uncompr is still mostly zeroes, so it should compress
|
||||
* very well:
|
||||
*/
|
||||
c_stream.next_in = uncompr;
|
||||
c_stream.avail_in = (uInt)uncomprLen;
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
if (c_stream.avail_in != 0) {
|
||||
fprintf(stderr, "deflate not greedy\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* Feed in already compressed data and switch to no compression: */
|
||||
deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY);
|
||||
c_stream.next_in = compr;
|
||||
c_stream.avail_in = (uInt)comprLen/2;
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
|
||||
/* Switch back to compressing mode: */
|
||||
deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED);
|
||||
c_stream.next_in = uncompr;
|
||||
c_stream.avail_in = (uInt)uncomprLen;
|
||||
err = deflate(&c_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
fprintf(stderr, "deflate should report Z_STREAM_END\n");
|
||||
exit(1);
|
||||
}
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflate() with large buffers
|
||||
*/
|
||||
void test_large_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen)
|
||||
{
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = (uInt)comprLen;
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
for (;;) {
|
||||
d_stream.next_out = uncompr; /* discard the output */
|
||||
d_stream.avail_out = (uInt)uncomprLen;
|
||||
err = inflate(&d_stream, Z_NO_FLUSH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
CHECK_ERR(err, "large inflate");
|
||||
}
|
||||
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
if (d_stream.total_out != 2*uncomprLen + comprLen/2) {
|
||||
fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out);
|
||||
exit(1);
|
||||
} else {
|
||||
printf("large_inflate(): OK\n");
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with full flush
|
||||
*/
|
||||
void test_flush(Byte *compr, uLong comprLen)
|
||||
{
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
uInt len = (uInt)strlen(hello)+1;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
c_stream.next_in = (z_const unsigned char *)hello;
|
||||
c_stream.next_out = compr;
|
||||
c_stream.avail_in = 3;
|
||||
c_stream.avail_out = (uInt)*comprLen;
|
||||
err = deflate(&c_stream, Z_FULL_FLUSH);
|
||||
CHECK_ERR(err, "deflate");
|
||||
|
||||
compr[3]++; /* force an error in first compressed block */
|
||||
c_stream.avail_in = len - 3;
|
||||
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
CHECK_ERR(err, "deflate");
|
||||
}
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
|
||||
*comprLen = c_stream.total_out;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflateSync()
|
||||
*/
|
||||
void test_sync(Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen)
|
||||
{
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = 2; /* just read the zlib header */
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
d_stream.next_out = uncompr;
|
||||
d_stream.avail_out = (uInt)uncomprLen;
|
||||
|
||||
inflate(&d_stream, Z_NO_FLUSH);
|
||||
CHECK_ERR(err, "inflate");
|
||||
|
||||
d_stream.avail_in = (uInt)comprLen-2; /* read all compressed data */
|
||||
err = inflateSync(&d_stream); /* but skip the damaged part */
|
||||
CHECK_ERR(err, "inflateSync");
|
||||
|
||||
err = inflate(&d_stream, Z_FINISH);
|
||||
if (err != Z_DATA_ERROR) {
|
||||
fprintf(stderr, "inflate should report DATA_ERROR\n");
|
||||
/* Because of incorrect adler32 */
|
||||
exit(1);
|
||||
}
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
printf("after inflateSync(): hel%s\n", (char *)uncompr);
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test deflate() with preset dictionary
|
||||
*/
|
||||
void test_dict_deflate(Byte *compr, uLong comprLen)
|
||||
{
|
||||
z_stream c_stream; /* compression stream */
|
||||
int err;
|
||||
|
||||
c_stream.zalloc = zalloc;
|
||||
c_stream.zfree = zfree;
|
||||
c_stream.opaque = (voidpf)0;
|
||||
|
||||
err = deflateInit(&c_stream, Z_BEST_COMPRESSION);
|
||||
CHECK_ERR(err, "deflateInit");
|
||||
|
||||
err = deflateSetDictionary(&c_stream,
|
||||
(const Bytef*)dictionary, (int)sizeof(dictionary));
|
||||
CHECK_ERR(err, "deflateSetDictionary");
|
||||
|
||||
dictId = c_stream.adler;
|
||||
c_stream.next_out = compr;
|
||||
c_stream.avail_out = (uInt)comprLen;
|
||||
|
||||
c_stream.next_in = (z_const unsigned char *)hello;
|
||||
c_stream.avail_in = (uInt)strlen(hello)+1;
|
||||
|
||||
err = deflate(&c_stream, Z_FINISH);
|
||||
if (err != Z_STREAM_END) {
|
||||
fprintf(stderr, "deflate should report Z_STREAM_END\n");
|
||||
exit(1);
|
||||
}
|
||||
err = deflateEnd(&c_stream);
|
||||
CHECK_ERR(err, "deflateEnd");
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Test inflate() with a preset dictionary
|
||||
*/
|
||||
void test_dict_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
|
||||
uLong uncomprLen)
|
||||
{
|
||||
int err;
|
||||
z_stream d_stream; /* decompression stream */
|
||||
|
||||
strcpy((char*)uncompr, "garbage");
|
||||
|
||||
d_stream.zalloc = zalloc;
|
||||
d_stream.zfree = zfree;
|
||||
d_stream.opaque = (voidpf)0;
|
||||
|
||||
d_stream.next_in = compr;
|
||||
d_stream.avail_in = (uInt)comprLen;
|
||||
|
||||
err = inflateInit(&d_stream);
|
||||
CHECK_ERR(err, "inflateInit");
|
||||
|
||||
d_stream.next_out = uncompr;
|
||||
d_stream.avail_out = (uInt)uncomprLen;
|
||||
|
||||
for (;;) {
|
||||
err = inflate(&d_stream, Z_NO_FLUSH);
|
||||
if (err == Z_STREAM_END) break;
|
||||
if (err == Z_NEED_DICT) {
|
||||
if (d_stream.adler != dictId) {
|
||||
fprintf(stderr, "unexpected dictionary");
|
||||
exit(1);
|
||||
}
|
||||
err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary,
|
||||
(int)sizeof(dictionary));
|
||||
}
|
||||
CHECK_ERR(err, "inflate with dict");
|
||||
}
|
||||
|
||||
err = inflateEnd(&d_stream);
|
||||
CHECK_ERR(err, "inflateEnd");
|
||||
|
||||
if (strcmp((char*)uncompr, hello)) {
|
||||
fprintf(stderr, "bad inflate with dict\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("inflate with dictionary: %s\n", (char *)uncompr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Usage: example [output.gz [input.gz]]
|
||||
*/
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
Byte *compr, *uncompr;
|
||||
uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */
|
||||
uLong uncomprLen = comprLen;
|
||||
static const char* myVersion = ZLIB_VERSION;
|
||||
|
||||
if (zlibVersion()[0] != myVersion[0]) {
|
||||
fprintf(stderr, "incompatible zlib version\n");
|
||||
exit(1);
|
||||
|
||||
} else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) {
|
||||
fprintf(stderr, "warning: different zlib version\n");
|
||||
}
|
||||
|
||||
printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n",
|
||||
ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags());
|
||||
|
||||
compr = (Byte*)calloc((uInt)comprLen, 1);
|
||||
uncompr = (Byte*)calloc((uInt)uncomprLen, 1);
|
||||
/* compr and uncompr are cleared to avoid reading uninitialized
|
||||
* data and to ensure that uncompr compresses well.
|
||||
*/
|
||||
if (compr == Z_NULL || uncompr == Z_NULL) {
|
||||
printf("out of memory\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#ifdef Z_SOLO
|
||||
argc = strlen(argv[0]);
|
||||
#else
|
||||
test_compress(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
test_gzio((argc > 1 ? argv[1] : TESTFILE),
|
||||
uncompr, uncomprLen);
|
||||
#endif
|
||||
|
||||
test_deflate(compr, comprLen);
|
||||
test_inflate(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
test_large_deflate(compr, comprLen, uncompr, uncomprLen);
|
||||
test_large_inflate(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
test_flush(compr, &comprLen);
|
||||
test_sync(compr, comprLen, uncompr, uncomprLen);
|
||||
comprLen = uncomprLen;
|
||||
|
||||
test_dict_deflate(compr, comprLen);
|
||||
test_dict_inflate(compr, comprLen, uncompr, uncomprLen);
|
||||
|
||||
free(compr);
|
||||
free(uncompr);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,254 +0,0 @@
|
||||
/* fitblk.c contains minimal changes required to be compiled with zlibWrapper:
|
||||
* - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h"
|
||||
* - writing block to stdout was disabled */
|
||||
|
||||
/* fitblk.c: example of fitting compressed output to a specified size
|
||||
Not copyrighted -- provided to the public domain
|
||||
Version 1.1 25 November 2004 Mark Adler */
|
||||
|
||||
/* Version history:
|
||||
1.0 24 Nov 2004 First version
|
||||
1.1 25 Nov 2004 Change deflateInit2() to deflateInit()
|
||||
Use fixed-size, stack-allocated raw buffers
|
||||
Simplify code moving compression to subroutines
|
||||
Use assert() for internal errors
|
||||
Add detailed description of approach
|
||||
*/
|
||||
|
||||
/* Approach to just fitting a requested compressed size:
|
||||
|
||||
fitblk performs three compression passes on a portion of the input
|
||||
data in order to determine how much of that input will compress to
|
||||
nearly the requested output block size. The first pass generates
|
||||
enough deflate blocks to produce output to fill the requested
|
||||
output size plus a specified excess amount (see the EXCESS define
|
||||
below). The last deflate block may go quite a bit past that, but
|
||||
is discarded. The second pass decompresses and recompresses just
|
||||
the compressed data that fit in the requested plus excess sized
|
||||
buffer. The deflate process is terminated after that amount of
|
||||
input, which is less than the amount consumed on the first pass.
|
||||
The last deflate block of the result will be of a comparable size
|
||||
to the final product, so that the header for that deflate block and
|
||||
the compression ratio for that block will be about the same as in
|
||||
the final product. The third compression pass decompresses the
|
||||
result of the second step, but only the compressed data up to the
|
||||
requested size minus an amount to allow the compressed stream to
|
||||
complete (see the MARGIN define below). That will result in a
|
||||
final compressed stream whose length is less than or equal to the
|
||||
requested size. Assuming sufficient input and a requested size
|
||||
greater than a few hundred bytes, the shortfall will typically be
|
||||
less than ten bytes.
|
||||
|
||||
If the input is short enough that the first compression completes
|
||||
before filling the requested output size, then that compressed
|
||||
stream is return with no recompression.
|
||||
|
||||
EXCESS is chosen to be just greater than the shortfall seen in a
|
||||
two pass approach similar to the above. That shortfall is due to
|
||||
the last deflate block compressing more efficiently with a smaller
|
||||
header on the second pass. EXCESS is set to be large enough so
|
||||
that there is enough uncompressed data for the second pass to fill
|
||||
out the requested size, and small enough so that the final deflate
|
||||
block of the second pass will be close in size to the final deflate
|
||||
block of the third and final pass. MARGIN is chosen to be just
|
||||
large enough to assure that the final compression has enough room
|
||||
to complete in all cases.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
#include "zstd_zlibwrapper.h"
|
||||
|
||||
#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/
|
||||
#define local static
|
||||
|
||||
/* print nastygram and leave */
|
||||
local void quit(char *why)
|
||||
{
|
||||
fprintf(stderr, "fitblk abort: %s\n", why);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#define RAWLEN 4096 /* intermediate uncompressed buffer size */
|
||||
|
||||
/* compress from file to def until provided buffer is full or end of
|
||||
input reached; return last deflate() return value, or Z_ERRNO if
|
||||
there was read error on the file */
|
||||
local int partcompress(FILE *in, z_streamp def)
|
||||
{
|
||||
int ret, flush;
|
||||
unsigned char raw[RAWLEN];
|
||||
|
||||
flush = Z_SYNC_FLUSH;
|
||||
do {
|
||||
def->avail_in = (uInt)fread(raw, 1, RAWLEN, in);
|
||||
if (ferror(in))
|
||||
return Z_ERRNO;
|
||||
def->next_in = raw;
|
||||
if (feof(in))
|
||||
flush = Z_FINISH;
|
||||
LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
ret = deflate(def, flush);
|
||||
LOG_FITBLK("partcompress2 ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
} while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* recompress from inf's input to def's output; the input for inf and
|
||||
the output for def are set in those structures before calling;
|
||||
return last deflate() return value, or Z_MEM_ERROR if inflate()
|
||||
was not able to allocate enough memory when it needed to */
|
||||
local int recompress(z_streamp inf, z_streamp def)
|
||||
{
|
||||
int ret, flush;
|
||||
unsigned char raw[RAWLEN];
|
||||
|
||||
flush = Z_NO_FLUSH;
|
||||
LOG_FITBLK("recompress start\n");
|
||||
do {
|
||||
/* decompress */
|
||||
inf->avail_out = RAWLEN;
|
||||
inf->next_out = raw;
|
||||
LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
|
||||
ret = inflate(inf, Z_NO_FLUSH);
|
||||
LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
|
||||
assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
|
||||
ret != Z_NEED_DICT);
|
||||
if (ret == Z_MEM_ERROR)
|
||||
return ret;
|
||||
|
||||
/* compress what was decompressed until done or no room */
|
||||
def->avail_in = RAWLEN - inf->avail_out;
|
||||
def->next_in = raw;
|
||||
if (inf->avail_out != 0)
|
||||
flush = Z_FINISH;
|
||||
LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
ret = deflate(def, flush);
|
||||
LOG_FITBLK("recompress2deflate ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
} while (ret != Z_STREAM_END && def->avail_out != 0);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define EXCESS 256 /* empirically determined stream overage */
|
||||
#define MARGIN 8 /* amount to back off for completion */
|
||||
|
||||
/* compress from stdin to fixed-size block on stdout */
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret; /* return code */
|
||||
unsigned size; /* requested fixed output block size */
|
||||
unsigned have; /* bytes written by deflate() call */
|
||||
unsigned char *blk; /* intermediate and final stream */
|
||||
unsigned char *tmp; /* close to desired size stream */
|
||||
z_stream def, inf; /* zlib deflate and inflate states */
|
||||
|
||||
/* get requested output size */
|
||||
if (argc != 2)
|
||||
quit("need one argument: size of output block");
|
||||
ret = (int)strtol(argv[1], argv + 1, 10);
|
||||
if (argv[1][0] != 0)
|
||||
quit("argument must be a number");
|
||||
if (ret < 8) /* 8 is minimum zlib stream size */
|
||||
quit("need positive size of 8 or greater");
|
||||
size = (unsigned)ret;
|
||||
|
||||
printf("zlib version %s\n", ZLIB_VERSION);
|
||||
if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
|
||||
|
||||
/* allocate memory for buffers and compression engine */
|
||||
blk = (unsigned char*)malloc(size + EXCESS);
|
||||
def.zalloc = Z_NULL;
|
||||
def.zfree = Z_NULL;
|
||||
def.opaque = Z_NULL;
|
||||
ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
|
||||
if (ret != Z_OK || blk == NULL)
|
||||
quit("out of memory");
|
||||
|
||||
/* compress from stdin until output full, or no more input */
|
||||
def.avail_out = size + EXCESS;
|
||||
def.next_out = blk;
|
||||
LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
|
||||
ret = partcompress(stdin, &def);
|
||||
printf("partcompress total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
|
||||
if (ret == Z_ERRNO)
|
||||
quit("error reading input");
|
||||
|
||||
/* if it all fit, then size was undersubscribed -- done! */
|
||||
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
|
||||
/* write block to stdout */
|
||||
have = size + EXCESS - def.avail_out;
|
||||
/* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
* quit("error writing output"); */
|
||||
|
||||
/* clean up and print results to stderr */
|
||||
ret = deflateEnd(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
free(blk);
|
||||
fprintf(stderr,
|
||||
"%u bytes unused out of %u requested (all input)\n",
|
||||
size - have, size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* it didn't all fit -- set up for recompression */
|
||||
inf.zalloc = Z_NULL;
|
||||
inf.zfree = Z_NULL;
|
||||
inf.opaque = Z_NULL;
|
||||
inf.avail_in = 0;
|
||||
inf.next_in = Z_NULL;
|
||||
ret = inflateInit(&inf);
|
||||
tmp = (unsigned char*)malloc(size + EXCESS);
|
||||
if (ret != Z_OK || tmp == NULL)
|
||||
quit("out of memory");
|
||||
ret = deflateReset(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
|
||||
/* do first recompression close to the right amount */
|
||||
inf.avail_in = size + EXCESS;
|
||||
inf.next_in = blk;
|
||||
def.avail_out = size + EXCESS;
|
||||
def.next_out = tmp;
|
||||
LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
|
||||
ret = recompress(&inf, &def);
|
||||
LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
|
||||
if (ret == Z_MEM_ERROR)
|
||||
quit("out of memory");
|
||||
|
||||
/* set up for next recompression */
|
||||
ret = inflateReset(&inf);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
ret = deflateReset(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
|
||||
/* do second and final recompression (third compression) */
|
||||
inf.avail_in = size - MARGIN; /* assure stream will complete */
|
||||
inf.next_in = tmp;
|
||||
def.avail_out = size;
|
||||
def.next_out = blk;
|
||||
LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
|
||||
ret = recompress(&inf, &def);
|
||||
LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
|
||||
if (ret == Z_MEM_ERROR)
|
||||
quit("out of memory");
|
||||
assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
|
||||
|
||||
/* done -- write block to stdout */
|
||||
have = size - def.avail_out;
|
||||
/* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
* quit("error writing output"); */
|
||||
|
||||
/* clean up and print results to stderr */
|
||||
free(tmp);
|
||||
ret = inflateEnd(&inf);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
ret = deflateEnd(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
free(blk);
|
||||
fprintf(stderr,
|
||||
"%u bytes unused out of %u requested (%lu input)\n",
|
||||
size - have, size, def.total_in);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
/* fitblk.c: example of fitting compressed output to a specified size
|
||||
Not copyrighted -- provided to the public domain
|
||||
Version 1.1 25 November 2004 Mark Adler */
|
||||
|
||||
/* Version history:
|
||||
1.0 24 Nov 2004 First version
|
||||
1.1 25 Nov 2004 Change deflateInit2() to deflateInit()
|
||||
Use fixed-size, stack-allocated raw buffers
|
||||
Simplify code moving compression to subroutines
|
||||
Use assert() for internal errors
|
||||
Add detailed description of approach
|
||||
*/
|
||||
|
||||
/* Approach to just fitting a requested compressed size:
|
||||
|
||||
fitblk performs three compression passes on a portion of the input
|
||||
data in order to determine how much of that input will compress to
|
||||
nearly the requested output block size. The first pass generates
|
||||
enough deflate blocks to produce output to fill the requested
|
||||
output size plus a specified excess amount (see the EXCESS define
|
||||
below). The last deflate block may go quite a bit past that, but
|
||||
is discarded. The second pass decompresses and recompresses just
|
||||
the compressed data that fit in the requested plus excess sized
|
||||
buffer. The deflate process is terminated after that amount of
|
||||
input, which is less than the amount consumed on the first pass.
|
||||
The last deflate block of the result will be of a comparable size
|
||||
to the final product, so that the header for that deflate block and
|
||||
the compression ratio for that block will be about the same as in
|
||||
the final product. The third compression pass decompresses the
|
||||
result of the second step, but only the compressed data up to the
|
||||
requested size minus an amount to allow the compressed stream to
|
||||
complete (see the MARGIN define below). That will result in a
|
||||
final compressed stream whose length is less than or equal to the
|
||||
requested size. Assuming sufficient input and a requested size
|
||||
greater than a few hundred bytes, the shortfall will typically be
|
||||
less than ten bytes.
|
||||
|
||||
If the input is short enough that the first compression completes
|
||||
before filling the requested output size, then that compressed
|
||||
stream is return with no recompression.
|
||||
|
||||
EXCESS is chosen to be just greater than the shortfall seen in a
|
||||
two pass approach similar to the above. That shortfall is due to
|
||||
the last deflate block compressing more efficiently with a smaller
|
||||
header on the second pass. EXCESS is set to be large enough so
|
||||
that there is enough uncompressed data for the second pass to fill
|
||||
out the requested size, and small enough so that the final deflate
|
||||
block of the second pass will be close in size to the final deflate
|
||||
block of the third and final pass. MARGIN is chosen to be just
|
||||
large enough to assure that the final compression has enough room
|
||||
to complete in all cases.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <assert.h>
|
||||
#include "zlib.h"
|
||||
|
||||
#define local static
|
||||
|
||||
/* print nastygram and leave */
|
||||
local void quit(char *why)
|
||||
{
|
||||
fprintf(stderr, "fitblk abort: %s\n", why);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#define RAWLEN 4096 /* intermediate uncompressed buffer size */
|
||||
|
||||
/* compress from file to def until provided buffer is full or end of
|
||||
input reached; return last deflate() return value, or Z_ERRNO if
|
||||
there was read error on the file */
|
||||
local int partcompress(FILE *in, z_streamp def)
|
||||
{
|
||||
int ret, flush;
|
||||
unsigned char raw[RAWLEN];
|
||||
|
||||
flush = Z_NO_FLUSH;
|
||||
do {
|
||||
def->avail_in = fread(raw, 1, RAWLEN, in);
|
||||
if (ferror(in))
|
||||
return Z_ERRNO;
|
||||
def->next_in = raw;
|
||||
if (feof(in))
|
||||
flush = Z_FINISH;
|
||||
ret = deflate(def, flush);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
} while (def->avail_out != 0 && flush == Z_NO_FLUSH);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* recompress from inf's input to def's output; the input for inf and
|
||||
the output for def are set in those structures before calling;
|
||||
return last deflate() return value, or Z_MEM_ERROR if inflate()
|
||||
was not able to allocate enough memory when it needed to */
|
||||
local int recompress(z_streamp inf, z_streamp def)
|
||||
{
|
||||
int ret, flush;
|
||||
unsigned char raw[RAWLEN];
|
||||
|
||||
flush = Z_NO_FLUSH;
|
||||
do {
|
||||
/* decompress */
|
||||
inf->avail_out = RAWLEN;
|
||||
inf->next_out = raw;
|
||||
ret = inflate(inf, Z_NO_FLUSH);
|
||||
assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
|
||||
ret != Z_NEED_DICT);
|
||||
if (ret == Z_MEM_ERROR)
|
||||
return ret;
|
||||
|
||||
/* compress what was decompressed until done or no room */
|
||||
def->avail_in = RAWLEN - inf->avail_out;
|
||||
def->next_in = raw;
|
||||
if (inf->avail_out != 0)
|
||||
flush = Z_FINISH;
|
||||
ret = deflate(def, flush);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
} while (ret != Z_STREAM_END && def->avail_out != 0);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define EXCESS 256 /* empirically determined stream overage */
|
||||
#define MARGIN 8 /* amount to back off for completion */
|
||||
|
||||
/* compress from stdin to fixed-size block on stdout */
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret; /* return code */
|
||||
unsigned size; /* requested fixed output block size */
|
||||
unsigned have; /* bytes written by deflate() call */
|
||||
unsigned char *blk; /* intermediate and final stream */
|
||||
unsigned char *tmp; /* close to desired size stream */
|
||||
z_stream def, inf; /* zlib deflate and inflate states */
|
||||
|
||||
/* get requested output size */
|
||||
if (argc != 2)
|
||||
quit("need one argument: size of output block");
|
||||
ret = strtol(argv[1], argv + 1, 10);
|
||||
if (argv[1][0] != 0)
|
||||
quit("argument must be a number");
|
||||
if (ret < 8) /* 8 is minimum zlib stream size */
|
||||
quit("need positive size of 8 or greater");
|
||||
size = (unsigned)ret;
|
||||
|
||||
/* allocate memory for buffers and compression engine */
|
||||
blk = malloc(size + EXCESS);
|
||||
def.zalloc = Z_NULL;
|
||||
def.zfree = Z_NULL;
|
||||
def.opaque = Z_NULL;
|
||||
ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
|
||||
if (ret != Z_OK || blk == NULL)
|
||||
quit("out of memory");
|
||||
|
||||
/* compress from stdin until output full, or no more input */
|
||||
def.avail_out = size + EXCESS;
|
||||
def.next_out = blk;
|
||||
ret = partcompress(stdin, &def);
|
||||
if (ret == Z_ERRNO)
|
||||
quit("error reading input");
|
||||
|
||||
/* if it all fit, then size was undersubscribed -- done! */
|
||||
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
|
||||
/* write block to stdout */
|
||||
have = size + EXCESS - def.avail_out;
|
||||
if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
quit("error writing output");
|
||||
|
||||
/* clean up and print results to stderr */
|
||||
ret = deflateEnd(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
free(blk);
|
||||
fprintf(stderr,
|
||||
"%u bytes unused out of %u requested (all input)\n",
|
||||
size - have, size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* it didn't all fit -- set up for recompression */
|
||||
inf.zalloc = Z_NULL;
|
||||
inf.zfree = Z_NULL;
|
||||
inf.opaque = Z_NULL;
|
||||
inf.avail_in = 0;
|
||||
inf.next_in = Z_NULL;
|
||||
ret = inflateInit(&inf);
|
||||
tmp = malloc(size + EXCESS);
|
||||
if (ret != Z_OK || tmp == NULL)
|
||||
quit("out of memory");
|
||||
ret = deflateReset(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
|
||||
/* do first recompression close to the right amount */
|
||||
inf.avail_in = size + EXCESS;
|
||||
inf.next_in = blk;
|
||||
def.avail_out = size + EXCESS;
|
||||
def.next_out = tmp;
|
||||
ret = recompress(&inf, &def);
|
||||
if (ret == Z_MEM_ERROR)
|
||||
quit("out of memory");
|
||||
|
||||
/* set up for next recompression */
|
||||
ret = inflateReset(&inf);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
ret = deflateReset(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
|
||||
/* do second and final recompression (third compression) */
|
||||
inf.avail_in = size - MARGIN; /* assure stream will complete */
|
||||
inf.next_in = tmp;
|
||||
def.avail_out = size;
|
||||
def.next_out = blk;
|
||||
ret = recompress(&inf, &def);
|
||||
if (ret == Z_MEM_ERROR)
|
||||
quit("out of memory");
|
||||
assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
|
||||
|
||||
/* done -- write block to stdout */
|
||||
have = size - def.avail_out;
|
||||
if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
|
||||
quit("error writing output");
|
||||
|
||||
/* clean up and print results to stderr */
|
||||
free(tmp);
|
||||
ret = inflateEnd(&inf);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
ret = deflateEnd(&def);
|
||||
assert(ret != Z_STREAM_ERROR);
|
||||
free(blk);
|
||||
fprintf(stderr,
|
||||
"%u bytes unused out of %u requested (%lu input)\n",
|
||||
size - have, size, def.total_in);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,605 +0,0 @@
|
||||
/* minigzip.c contains minimal changes required to be compiled with zlibWrapper:
|
||||
* - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" */
|
||||
|
||||
/* minigzip.c -- simulate gzip using the zlib compression library
|
||||
* Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly.
|
||||
* For conditions of distribution and use, see https://www.zlib.net/zlib_license.html
|
||||
*/
|
||||
|
||||
/*
|
||||
* minigzip is a minimal implementation of the gzip utility. This is
|
||||
* only an example of using zlib and isn't meant to replace the
|
||||
* full-featured gzip. No attempt is made to deal with file systems
|
||||
* limiting names to 14 or 8+3 characters, etc... Error checking is
|
||||
* very limited. So use minigzip only for testing; use gzip for the
|
||||
* real thing. On MSDOS, use only on file names without extension
|
||||
* or in pipe mode.
|
||||
*/
|
||||
|
||||
/* @(#) $Id$ */
|
||||
|
||||
#define _POSIX_SOURCE /* fileno */
|
||||
|
||||
#include "zstd_zlibwrapper.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef STDC
|
||||
# include <string.h>
|
||||
# include <stdlib.h>
|
||||
#endif
|
||||
|
||||
#ifdef USE_MMAP
|
||||
# include <sys/types.h>
|
||||
# include <sys/mman.h>
|
||||
# include <sys/stat.h>
|
||||
#endif
|
||||
|
||||
#if defined(MSDOS) || defined(OS2) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h>
|
||||
# include <io.h>
|
||||
# ifdef UNDER_CE
|
||||
# include <stdlib.h>
|
||||
# endif
|
||||
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
|
||||
#else
|
||||
# define SET_BINARY_MODE(file)
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
# define snprintf _snprintf
|
||||
#endif
|
||||
|
||||
#ifdef VMS
|
||||
# define unlink delete
|
||||
# define GZ_SUFFIX "-gz"
|
||||
#endif
|
||||
#ifdef RISCOS
|
||||
# define unlink remove
|
||||
# define GZ_SUFFIX "-gz"
|
||||
# define fileno(file) file->__file
|
||||
#endif
|
||||
#if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os
|
||||
# include <unix.h> /* for fileno */
|
||||
#endif
|
||||
|
||||
#if !defined(Z_HAVE_UNISTD_H) && !defined(_LARGEFILE64_SOURCE)
|
||||
#ifndef _WIN32 /* unlink already in stdio.h for WIN32 */
|
||||
extern int unlink _Z_OF((const char *));
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(UNDER_CE)
|
||||
# include <windows.h>
|
||||
# define perror(s) pwinerror(s)
|
||||
|
||||
/* Map the Windows error number in ERROR to a locale-dependent error
|
||||
message string and return a pointer to it. Typically, the values
|
||||
for ERROR come from GetLastError.
|
||||
|
||||
The string pointed to shall not be modified by the application,
|
||||
but may be overwritten by a subsequent call to strwinerror
|
||||
|
||||
The strwinerror function does not change the current setting
|
||||
of GetLastError. */
|
||||
|
||||
static char *strwinerror(DWORD error)
|
||||
{
|
||||
static char buf[1024];
|
||||
|
||||
wchar_t *msgbuf;
|
||||
DWORD lasterr = GetLastError();
|
||||
DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM
|
||||
| FORMAT_MESSAGE_ALLOCATE_BUFFER,
|
||||
NULL,
|
||||
error,
|
||||
0, /* Default language */
|
||||
(LPVOID)&msgbuf,
|
||||
0,
|
||||
NULL);
|
||||
if (chars != 0) {
|
||||
/* If there is an \r\n appended, zap it. */
|
||||
if (chars >= 2
|
||||
&& msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') {
|
||||
chars -= 2;
|
||||
msgbuf[chars] = 0;
|
||||
}
|
||||
|
||||
if (chars > sizeof (buf) - 1) {
|
||||
chars = sizeof (buf) - 1;
|
||||
msgbuf[chars] = 0;
|
||||
}
|
||||
|
||||
wcstombs(buf, msgbuf, chars + 1);
|
||||
LocalFree(msgbuf);
|
||||
}
|
||||
else {
|
||||
sprintf(buf, "unknown win32 error (%ld)", error);
|
||||
}
|
||||
|
||||
SetLastError(lasterr);
|
||||
return buf;
|
||||
}
|
||||
|
||||
static void pwinerror (const char *s)
|
||||
{
|
||||
if (s && *s)
|
||||
fprintf(stderr, "%s: %s\n", s, strwinerror(GetLastError ()));
|
||||
else
|
||||
fprintf(stderr, "%s\n", strwinerror(GetLastError ()));
|
||||
}
|
||||
|
||||
#endif /* UNDER_CE */
|
||||
|
||||
#ifndef GZ_SUFFIX
|
||||
# define GZ_SUFFIX ".gz"
|
||||
#endif
|
||||
#define SUFFIX_LEN (sizeof(GZ_SUFFIX)-1)
|
||||
|
||||
#define BUFLEN 16384
|
||||
#define MAX_NAME_LEN 1024
|
||||
|
||||
#ifdef MAXSEG_64K
|
||||
# define local static
|
||||
/* Needed for systems with limitation on stack size. */
|
||||
#else
|
||||
# define local
|
||||
#endif
|
||||
|
||||
#ifdef Z_SOLO
|
||||
/* for Z_SOLO, create simplified gz* functions using deflate and inflate */
|
||||
|
||||
#if defined(Z_HAVE_UNISTD_H) || defined(Z_LARGE)
|
||||
# include <unistd.h> /* for unlink() */
|
||||
#endif
|
||||
|
||||
void *myalloc _Z_OF((void *, unsigned, unsigned));
|
||||
void myfree _Z_OF((void *, void *));
|
||||
|
||||
void *myalloc(q, n, m)
|
||||
void *q;
|
||||
unsigned n, m;
|
||||
{
|
||||
q = Z_NULL;
|
||||
return calloc(n, m);
|
||||
}
|
||||
|
||||
void myfree(q, p)
|
||||
void *q, *p;
|
||||
{
|
||||
q = Z_NULL;
|
||||
free(p);
|
||||
}
|
||||
|
||||
typedef struct gzFile_s {
|
||||
FILE *file;
|
||||
int write;
|
||||
int err;
|
||||
char *msg;
|
||||
z_stream strm;
|
||||
} *gzFile;
|
||||
|
||||
gzFile gzopen _Z_OF((const char *, const char *));
|
||||
gzFile gzdopen _Z_OF((int, const char *));
|
||||
gzFile gz_open _Z_OF((const char *, int, const char *));
|
||||
|
||||
gzFile gzopen(path, mode)
|
||||
const char *path;
|
||||
const char *mode;
|
||||
{
|
||||
return gz_open(path, -1, mode);
|
||||
}
|
||||
|
||||
gzFile gzdopen(fd, mode)
|
||||
int fd;
|
||||
const char *mode;
|
||||
{
|
||||
return gz_open(NULL, fd, mode);
|
||||
}
|
||||
|
||||
gzFile gz_open(const char *path, int fd, const char *mode) {
|
||||
gzFile gz;
|
||||
int ret;
|
||||
|
||||
gz = malloc(sizeof(struct gzFile_s));
|
||||
if (gz == NULL)
|
||||
return NULL;
|
||||
gz->write = strchr(mode, 'w') != NULL;
|
||||
gz->strm.zalloc = myalloc;
|
||||
gz->strm.zfree = myfree;
|
||||
gz->strm.opaque = Z_NULL;
|
||||
if (gz->write)
|
||||
ret = deflateInit2(&(gz->strm), -1, 8, 15 + 16, 8, 0);
|
||||
else {
|
||||
gz->strm.next_in = 0;
|
||||
gz->strm.avail_in = Z_NULL;
|
||||
ret = inflateInit2(&(gz->strm), 15 + 16);
|
||||
}
|
||||
if (ret != Z_OK) {
|
||||
free(gz);
|
||||
return NULL;
|
||||
}
|
||||
gz->file = path == NULL ? fdopen(fd, gz->write ? "wb" : "rb") :
|
||||
fopen(path, gz->write ? "wb" : "rb");
|
||||
if (gz->file == NULL) {
|
||||
gz->write ? deflateEnd(&(gz->strm)) : inflateEnd(&(gz->strm));
|
||||
free(gz);
|
||||
return NULL;
|
||||
}
|
||||
gz->err = 0;
|
||||
gz->msg = "";
|
||||
return gz;
|
||||
}
|
||||
|
||||
int gzwrite _Z_OF((gzFile, const void *, unsigned));
|
||||
|
||||
int gzwrite(gzFile gz, const void *buf, unsigned len) {
|
||||
z_stream *strm;
|
||||
unsigned char out[BUFLEN] = { 0 };
|
||||
|
||||
if (gz == NULL || !gz->write)
|
||||
return 0;
|
||||
strm = &(gz->strm);
|
||||
strm->next_in = (void *)buf;
|
||||
strm->avail_in = len;
|
||||
do {
|
||||
strm->next_out = out;
|
||||
strm->avail_out = BUFLEN;
|
||||
(void)deflate(strm, Z_NO_FLUSH);
|
||||
fwrite(out, 1, BUFLEN - strm->avail_out, gz->file);
|
||||
} while (strm->avail_out == 0);
|
||||
return len;
|
||||
}
|
||||
|
||||
int gzread _Z_OF((gzFile, void *, unsigned));
|
||||
|
||||
int gzread(gzFile gz, void *buf, unsigned len) {
|
||||
int ret;
|
||||
unsigned got;
|
||||
unsigned char in[1];
|
||||
z_stream *strm;
|
||||
|
||||
if (gz == NULL || gz->write)
|
||||
return 0;
|
||||
if (gz->err)
|
||||
return 0;
|
||||
strm = &(gz->strm);
|
||||
strm->next_out = (void *)buf;
|
||||
strm->avail_out = len;
|
||||
do {
|
||||
got = fread(in, 1, 1, gz->file);
|
||||
if (got == 0)
|
||||
break;
|
||||
strm->next_in = in;
|
||||
strm->avail_in = 1;
|
||||
ret = inflate(strm, Z_NO_FLUSH);
|
||||
if (ret == Z_DATA_ERROR) {
|
||||
gz->err = Z_DATA_ERROR;
|
||||
gz->msg = strm->msg;
|
||||
return 0;
|
||||
}
|
||||
if (ret == Z_STREAM_END)
|
||||
inflateReset(strm);
|
||||
} while (strm->avail_out);
|
||||
return len - strm->avail_out;
|
||||
}
|
||||
|
||||
int gzclose _Z_OF((gzFile));
|
||||
|
||||
int gzclose(gzFile gz) {
|
||||
z_stream *strm;
|
||||
unsigned char out[BUFLEN] = { 0 };
|
||||
|
||||
if (gz == NULL)
|
||||
return Z_STREAM_ERROR;
|
||||
strm = &(gz->strm);
|
||||
if (gz->write) {
|
||||
strm->next_in = Z_NULL;
|
||||
strm->avail_in = 0;
|
||||
do {
|
||||
strm->next_out = out;
|
||||
strm->avail_out = BUFLEN;
|
||||
(void)deflate(strm, Z_FINISH);
|
||||
fwrite(out, 1, BUFLEN - strm->avail_out, gz->file);
|
||||
} while (strm->avail_out == 0);
|
||||
deflateEnd(strm);
|
||||
}
|
||||
else
|
||||
inflateEnd(strm);
|
||||
fclose(gz->file);
|
||||
free(gz);
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
const char *gzerror _Z_OF((gzFile, int *));
|
||||
|
||||
const char *gzerror(gzFile gz, int *err)
|
||||
{
|
||||
*err = gz->err;
|
||||
return gz->msg;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
char *prog;
|
||||
|
||||
void error _Z_OF((const char *msg));
|
||||
void gz_compress _Z_OF((FILE *in, gzFile out));
|
||||
#ifdef USE_MMAP
|
||||
int gz_compress_mmap _Z_OF((FILE *in, gzFile out));
|
||||
#endif
|
||||
void gz_uncompress _Z_OF((gzFile in, FILE *out));
|
||||
void file_compress _Z_OF((char *file, char *mode));
|
||||
void file_uncompress _Z_OF((char *file));
|
||||
int main _Z_OF((int argc, char *argv[]));
|
||||
|
||||
/* ===========================================================================
|
||||
* Display error message and exit
|
||||
*/
|
||||
void error(const char *msg)
|
||||
{
|
||||
fprintf(stderr, "%s: %s\n", prog, msg);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
* Compress input to output then close both files.
|
||||
*/
|
||||
|
||||
void gz_compress(FILE *in, gzFile out)
|
||||
{
|
||||
local char buf[BUFLEN];
|
||||
int len;
|
||||
int err;
|
||||
|
||||
#ifdef USE_MMAP
|
||||
/* Try first compressing with mmap. If mmap fails (minigzip used in a
|
||||
* pipe), use the normal fread loop.
|
||||
*/
|
||||
if (gz_compress_mmap(in, out) == Z_OK) return;
|
||||
#endif
|
||||
for (;;) {
|
||||
len = (int)fread(buf, 1, sizeof(buf), in);
|
||||
if (ferror(in)) {
|
||||
perror("fread");
|
||||
exit(1);
|
||||
}
|
||||
if (len == 0) break;
|
||||
|
||||
if (gzwrite(out, buf, (unsigned)len) != len) error(gzerror(out, &err));
|
||||
}
|
||||
fclose(in);
|
||||
if (gzclose(out) != Z_OK) error("failed gzclose");
|
||||
}
|
||||
|
||||
#ifdef USE_MMAP /* MMAP version, Miguel Albrecht <[email protected]> */
|
||||
|
||||
/* Try compressing the input file at once using mmap. Return Z_OK if
|
||||
* if success, Z_ERRNO otherwise.
|
||||
*/
|
||||
int gz_compress_mmap(FILE *in, gzFile out) {
|
||||
int len;
|
||||
int err;
|
||||
int ifd = fileno(in);
|
||||
caddr_t buf; /* mmap'ed buffer for the entire input file */
|
||||
off_t buf_len; /* length of the input file */
|
||||
struct stat sb;
|
||||
|
||||
/* Determine the size of the file, needed for mmap: */
|
||||
if (fstat(ifd, &sb) < 0) return Z_ERRNO;
|
||||
buf_len = sb.st_size;
|
||||
if (buf_len <= 0) return Z_ERRNO;
|
||||
|
||||
/* Now do the actual mmap: */
|
||||
buf = mmap((caddr_t) 0, buf_len, PROT_READ, MAP_SHARED, ifd, (off_t)0);
|
||||
if (buf == (caddr_t)(-1)) return Z_ERRNO;
|
||||
|
||||
/* Compress the whole file at once: */
|
||||
len = gzwrite(out, (char *)buf, (unsigned)buf_len);
|
||||
|
||||
if (len != (int)buf_len) error(gzerror(out, &err));
|
||||
|
||||
munmap(buf, buf_len);
|
||||
fclose(in);
|
||||
if (gzclose(out) != Z_OK) error("failed gzclose");
|
||||
return Z_OK;
|
||||
}
|
||||
#endif /* USE_MMAP */
|
||||
|
||||
/* ===========================================================================
|
||||
* Uncompress input to output then close both files.
|
||||
*/
|
||||
void gz_uncompress(gzFile in, FILE *out) {
|
||||
local char buf[BUFLEN];
|
||||
int len;
|
||||
int err;
|
||||
|
||||
for (;;) {
|
||||
len = gzread(in, buf, sizeof(buf));
|
||||
if (len < 0) error (gzerror(in, &err));
|
||||
if (len == 0) break;
|
||||
|
||||
if ((int)fwrite(buf, 1, (unsigned)len, out) != len) {
|
||||
error("failed fwrite");
|
||||
}
|
||||
}
|
||||
if (fclose(out)) error("failed fclose");
|
||||
|
||||
if (gzclose(in) != Z_OK) error("failed gzclose");
|
||||
}
|
||||
|
||||
|
||||
/* ===========================================================================
|
||||
* Compress the given file: create a corresponding .gz file and remove the
|
||||
* original.
|
||||
*/
|
||||
void file_compress(char *file, char *mode) {
|
||||
local char outfile[MAX_NAME_LEN];
|
||||
FILE *in;
|
||||
gzFile out;
|
||||
|
||||
if (strlen(file) + strlen(GZ_SUFFIX) >= sizeof(outfile)) {
|
||||
fprintf(stderr, "%s: filename too long\n", prog);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
strcpy(outfile, file);
|
||||
strcat(outfile, GZ_SUFFIX);
|
||||
|
||||
in = fopen(file, "rb");
|
||||
if (in == NULL) {
|
||||
perror(file);
|
||||
exit(1);
|
||||
}
|
||||
out = gzopen(outfile, mode);
|
||||
if (out == NULL) {
|
||||
fprintf(stderr, "%s: can't gzopen %s\n", prog, outfile);
|
||||
exit(1);
|
||||
}
|
||||
gz_compress(in, out);
|
||||
|
||||
unlink(file);
|
||||
}
|
||||
|
||||
|
||||
/* ===========================================================================
|
||||
* Uncompress the given file and remove the original.
|
||||
*/
|
||||
void file_uncompress(char *file) {
|
||||
local char buf[MAX_NAME_LEN];
|
||||
char *infile, *outfile;
|
||||
FILE *out;
|
||||
gzFile in;
|
||||
size_t len = strlen(file);
|
||||
|
||||
if (len + strlen(GZ_SUFFIX) >= sizeof(buf)) {
|
||||
fprintf(stderr, "%s: filename too long\n", prog);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
strcpy(buf, file);
|
||||
|
||||
if (len > SUFFIX_LEN && strcmp(file+len-SUFFIX_LEN, GZ_SUFFIX) == 0) {
|
||||
infile = file;
|
||||
outfile = buf;
|
||||
outfile[len-3] = '\0';
|
||||
} else {
|
||||
outfile = file;
|
||||
infile = buf;
|
||||
strcat(infile, GZ_SUFFIX);
|
||||
}
|
||||
in = gzopen(infile, "rb");
|
||||
if (in == NULL) {
|
||||
fprintf(stderr, "%s: can't gzopen %s\n", prog, infile);
|
||||
exit(1);
|
||||
}
|
||||
out = fopen(outfile, "wb");
|
||||
if (out == NULL) {
|
||||
perror(file);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
gz_uncompress(in, out);
|
||||
|
||||
unlink(infile);
|
||||
}
|
||||
|
||||
|
||||
/* ===========================================================================
|
||||
* Usage: minigzip [-c] [-d] [-f] [-h] [-r] [-1 to -9] [files...]
|
||||
* -c : write to standard output
|
||||
* -d : decompress
|
||||
* -f : compress with Z_FILTERED
|
||||
* -h : compress with Z_HUFFMAN_ONLY
|
||||
* -r : compress with Z_RLE
|
||||
* -1 to -9 : compression level
|
||||
*/
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
int copyout = 0;
|
||||
int uncompr = 0;
|
||||
gzFile file;
|
||||
char *bname, outmode[20];
|
||||
|
||||
strcpy(outmode, "wb6 ");
|
||||
|
||||
prog = argv[0];
|
||||
bname = strrchr(argv[0], '/');
|
||||
if (bname)
|
||||
bname++;
|
||||
else
|
||||
bname = argv[0];
|
||||
argc--, argv++;
|
||||
|
||||
if (!strcmp(bname, "gunzip"))
|
||||
uncompr = 1;
|
||||
else if (!strcmp(bname, "zcat"))
|
||||
copyout = uncompr = 1;
|
||||
|
||||
while (argc > 0) {
|
||||
if (strcmp(*argv, "-c") == 0)
|
||||
copyout = 1;
|
||||
else if (strcmp(*argv, "-d") == 0)
|
||||
uncompr = 1;
|
||||
else if (strcmp(*argv, "-f") == 0)
|
||||
outmode[3] = 'f';
|
||||
else if (strcmp(*argv, "-h") == 0)
|
||||
outmode[3] = 'h';
|
||||
else if (strcmp(*argv, "-r") == 0)
|
||||
outmode[3] = 'R';
|
||||
else if ((*argv)[0] == '-' && (*argv)[1] >= '1' && (*argv)[1] <= '9' &&
|
||||
(*argv)[2] == 0)
|
||||
outmode[2] = (*argv)[1];
|
||||
else
|
||||
break;
|
||||
argc--, argv++;
|
||||
}
|
||||
if (outmode[3] == ' ')
|
||||
outmode[3] = 0;
|
||||
if (argc == 0) {
|
||||
SET_BINARY_MODE(stdin);
|
||||
SET_BINARY_MODE(stdout);
|
||||
if (uncompr) {
|
||||
file = gzdopen(fileno(stdin), "rb");
|
||||
if (file == NULL) error("can't gzdopen stdin");
|
||||
gz_uncompress(file, stdout);
|
||||
} else {
|
||||
file = gzdopen(fileno(stdout), outmode);
|
||||
if (file == NULL) error("can't gzdopen stdout");
|
||||
gz_compress(stdin, file);
|
||||
}
|
||||
} else {
|
||||
if (copyout) {
|
||||
SET_BINARY_MODE(stdout);
|
||||
}
|
||||
do {
|
||||
if (uncompr) {
|
||||
if (copyout) {
|
||||
file = gzopen(*argv, "rb");
|
||||
if (file == NULL)
|
||||
fprintf(stderr, "%s: can't gzopen %s\n", prog, *argv);
|
||||
else
|
||||
gz_uncompress(file, stdout);
|
||||
} else {
|
||||
file_uncompress(*argv);
|
||||
}
|
||||
} else {
|
||||
if (copyout) {
|
||||
FILE * in = fopen(*argv, "rb");
|
||||
|
||||
if (in == NULL) {
|
||||
perror(*argv);
|
||||
} else {
|
||||
file = gzdopen(fileno(stdout), outmode);
|
||||
if (file == NULL) error("can't gzdopen stdout");
|
||||
|
||||
gz_compress(in, file);
|
||||
}
|
||||
|
||||
} else {
|
||||
file_compress(*argv, outmode);
|
||||
}
|
||||
}
|
||||
} while (argv++, --argc);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user