jeanlemotan/FS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added embedded libzip

Browse Source 
Prefixed all libzip zip_ funcs to fs_zip_ to avoid clashes and link errors due to assimp
Zip pack and zip writer work with libzip now
This commit was merged in pull request #2.
This commit is contained in:
catalinvasile
2024-07-16 12:57:03 +02:00
parent 5ec1fddf57
commit 56c367a3c4
144 changed files with 16978 additions and 1330 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/fs/zip/DeflateFileWriter.h
-37
View File
@@ -1,37 +0,0 @@
#pragma once
#include "fs/AbsPath.h"
#include "fs/zip/ZipWriter.h"
#include "tl/ptr.h"
namespace fs
{
class IFilesystem;
//////////////////////////////////////////////////////////////////////////
class FS_API DeflateFileWriter
{
public:
//compression level: 0 to 9
DeflateFileWriter(AbsPathView filePath, tl::lent_ref<IFilesystem> filesystem, uint8_t compressionLevel);
DeflateFileWriter(const DeflateFileWriter&) = delete;
DeflateFileWriter& operator=(const DeflateFileWriter&) = delete;
DeflateFileWriter(DeflateFileWriter&&) = default;
DeflateFileWriter& operator=(DeflateFileWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& sink);
private:
AbsPath m_filePath;
tl::lent_ref<IFilesystem> m_filesystem;
uint8_t m_compressionLevel = 9;
};
//////////////////////////////////////////////////////////////////////////
}
include/fs/zip/DeflateSinkWriter.h
-37
View File
@@ -1,37 +0,0 @@
#pragma once
#include "fs/IStreamSource.h"
#include "fs/zip/ZipWriter.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class ISource;
//////////////////////////////////////////////////////////////////////////
class FS_API DeflateSinkWriter
{
public:
//compression level: 0 to 9
DeflateSinkWriter(IStreamSource& source, uint8_t compressionLevel);
DeflateSinkWriter(tl::span<const uint8_t> data, uint8_t compressionLevel);
DeflateSinkWriter(const DeflateSinkWriter&) = delete;
DeflateSinkWriter& operator=(const DeflateSinkWriter&) = delete;
DeflateSinkWriter(DeflateSinkWriter&&) = default;
DeflateSinkWriter& operator=(DeflateSinkWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& sink);
private:
IStreamSource* m_source = nullptr;
tl::span<const uint8_t> m_data;
uint8_t m_compressionLevel = 9;
};
//////////////////////////////////////////////////////////////////////////
}
include/fs/zip/StoreFileWriter.h
-35
View File
@@ -1,35 +0,0 @@
#pragma once
#include "fs/AbsPath.h"
#include "fs/zip/ZipWriter.h"
#include "tl/ptr.h"
namespace fs
{
class IFilesystem;
//////////////////////////////////////////////////////////////////////////
class FS_API StoreFileWriter
{
public:
StoreFileWriter(AbsPathView filePath, tl::lent_ref<IFilesystem> filesystem);
StoreFileWriter(const StoreFileWriter&) = delete;
StoreFileWriter& operator=(const StoreFileWriter&) = delete;
StoreFileWriter(StoreFileWriter&&) = default;
StoreFileWriter& operator=(StoreFileWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& sink);
private:
AbsPath m_filePath;
tl::lent_ref<IFilesystem> m_filesystem;
};
//////////////////////////////////////////////////////////////////////////
}
include/fs/zip/StoreSinkWriter.h
-32
View File
@@ -1,32 +0,0 @@
#pragma once
#include "fs/zip/ZipWriter.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class ISource;
//////////////////////////////////////////////////////////////////////////
class FS_API StoreSinkWriter
{
public:
explicit StoreSinkWriter(ISource& source);
StoreSinkWriter(const StoreSinkWriter&) = delete;
StoreSinkWriter& operator=(const StoreSinkWriter&) = delete;
StoreSinkWriter(StoreSinkWriter&&) = default;
StoreSinkWriter& operator=(StoreSinkWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& sink);
private:
tl::reference_wrapper<ISource> m_inSource;
};
//////////////////////////////////////////////////////////////////////////
}
include/fs/zip/ZipBase.h
-158
View File
@@ -1,158 +0,0 @@
#pragma once
#include "tl/flag_set.h"
#include "fs/Api.h"
namespace fs
{
class FS_API ZipBase
{
public:
enum class GeneralBitFlag : uint16_t
{
Encrypted = 1 << 0,
CompressionMethodSpecific1 = 1 << 1,
CompressionMethodSpecific2 = 1 << 2,
DataDescriptorNeeded = 1 << 3,
StrongEncryption = 1 << 6,
LanguageEncodingFlag = 1 << 11,
LocalHeaderFieldsMasked = 1 << 13,
};
typedef tl::flag_set<GeneralBitFlag> GeneralBitFlags;
enum class CompressionMethod : uint16_t
{
Store = 0,
Shrunk = 1,
Reduced1 = 2,
Reduced2 = 3,
Reduced3 = 4,
Reduced4 = 5,
Implode = 6,
//Reserved = 7,
Deflate = 8,
Deflate64 = 9,
PKWareImplode = 10,
//Reserved = 11,
BZip2 = 12,
//Reserved = 13,
LZMA = 14,
//Reserved = 15,
//Reserved = 16,
//Reserved = 17,
Terse = 18,
LZ77 = 19,
LZ4 = 20, // non-standard
ZStd = 21, // non-standard
WavPack = 97,
PPMd = 98,
};
protected:
//////////////////////////////////////////////////////////////////////////
//These structs follow pretty closely the https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT standard
#pragma pack(push, 1)
struct LocalFileHeader
{
static constexpr uint32_t k_signature = 0x04034b50u;
uint32_t signature = k_signature;
uint16_t versionNeededToExtract = 0;
uint16_t generalBitFlag = 0;
uint16_t compressionMethod = 0;
uint16_t lastModFileTime = 0;
uint16_t lastModFileDate = 0;
uint32_t crc32 = 0;
uint32_t compressedSize = 0;
uint32_t uncompressedSize = 0;
uint16_t filenameLength = 0;
uint16_t extraFieldLength = 0;
};
struct DataDescriptor
{
uint32_t crc32 = 0;
uint32_t compressedSize = 0;
uint32_t uncompressedSize = 0;
};
struct CentralDirectoryFileHeader
{
static constexpr uint32_t k_signature = 0x02014b50u;
uint32_t signature = k_signature;
uint16_t versionMadeBy = 0;
uint16_t versionNeededToExtract = 0;
uint16_t generalBitFlag = 0;
uint16_t compressionMethod = 0;
uint16_t lastModFileTime = 0;
uint16_t lastModFileDate = 0;
uint32_t crc32 = 0;
uint32_t compressedSize = 0;
uint32_t uncompressedSize = 0;
uint16_t filenameLength = 0;
uint16_t extraFieldLength = 0;
uint16_t fileCommentLength = 0;
uint16_t diskNumberStart = 0;
uint16_t internalFileAttributes = 0;
uint32_t externalFileAttributes = 0;
uint32_t localHeaderOffset = 0;
};
struct EndOfCentralDirectoryRecord
{
static constexpr uint32_t k_signature = 0x06054b50u;
uint32_t signature = k_signature;
uint16_t diskNumber = 0;
uint16_t centralDirectoryStartDiskNumber = 0;
uint16_t centralDirectoryRecordCountOnThisDisk = 0;
uint16_t centralDirectoryRecordCount = 0;
uint32_t centralDirectorySize = 0;
uint32_t centralDirectoryOffset = 0;
uint16_t zipFileCommentLength = 0;
};
struct ExtendedInformationExtraField64
{
static constexpr uint32_t k_tag = 0x0001;
uint16_t tag = k_tag;
uint16_t size = 0;
uint64_t originalSize = 0;
uint64_t compressedSize = 0;
uint64_t localHeaderOffset = 0;
uint32_t diskNumber = 0;
};
struct EndOfCentralDirectoryRecord64
{
static constexpr uint32_t k_signature = 0x06064b50u;
uint32_t signature = k_signature;
uint64_t endOfCentralDirectorySize = 0;
uint16_t versionMadeBy = 0;
uint16_t versionNeededToExtract = 0;
uint32_t diskNumber = 0;
uint32_t centralDirectoryStartDiskNumber = 0;
uint64_t centralDirectoryRecordCountOnThisDisk = 0;
uint64_t centralDirectoryRecordCount = 0;
uint64_t centralDirectorySize = 0;
uint64_t centralDirectoryOffset = 0;
};
struct EndOfCentralDirectoryLocator64
{
static constexpr uint32_t k_signature = 0x07064b50u;
uint32_t signature = k_signature;
uint32_t centralDirectoryStartDiskNumber = 0;
uint64_t endOfCentralDirectoryRecordOffset = 0;
uint32_t diskCount = 0;
};
#pragma pack(pop)
static void computeMsDosTime(time_t time, uint16_t& o_date, uint16_t& o_time);
static CentralDirectoryFileHeader computeCentralDirectoryFileHeader(const LocalFileHeader& header, bool zip64);
};
}
include/fs/zip/ZipPack.h
+50 -27
View File
@@ -3,18 +3,40 @@
#include "tl/result.h"
#include "tl/string.h"
#include "tl/vector_map.h"
#include "fs/IStreamSource.h"
#include "fs/IPack.h"
#include "fs/AbsPath.h"
#include "fs/zip/ZipReader.h"
#include "fs/Api.h"
struct zip;
struct zip_source;
struct zip_file;
namespace tl
{
template<> struct ptr_traits<zip>
{
static constexpr inline bool custom_deleter = true;
static constexpr inline bool requires_virtual_destructor_check = false;
};
//template<> struct ptr_traits<zip_source>
//{
// static constexpr inline bool custom_deleter = true;
// static constexpr inline bool requires_virtual_destructor_check = false;
//};
template<> struct ptr_traits<zip_file>
{
static constexpr inline bool custom_deleter = true;
static constexpr inline bool requires_virtual_destructor_check = false;
};
}
namespace fs
{
class MapSourceView;
class FS_API ZipPack final : virtual public IPack
class FS_API ZipPack final : public IPack
{
public:
ZipPack() = default;
@@ -22,10 +44,10 @@ public:
typedef tl::result<tl::unique_ref<ZipPack>, Error> CreateResult;
static CreateResult create(tl::unique_ref<IMapSource> zipFileSource);
static CreateResult create(tl::lent_ref<const IFilesystem> filesystem, AbsPathView zipFileLocation);
static CreateResult create(tl::unique_ref<IMapSource> source);
static CreateResult create(tl::lent_ref<const IFilesystem> filesystem, AbsPathView path);
virtual void setEncryptionData(const tl::string& key, uint32_t rounds);
void setEncryptionData(const tl::string& key, uint32_t rounds);
OpenSourceResult openSource(AbsPathView path, SourceFlags flags = SourceFlags()) const override;
OpenStreamSourceResult openStreamSource(AbsPathView path, SourceFlags flags = SourceFlags()) const override;
@@ -42,46 +64,51 @@ public:
tl::result<AbsPath, Error> convertToNativePath(AbsPathView path) const override;
struct ZipData
{
tl::unique_ptr<IMapSource> fsMapSource;
tl::lent_ptr<const IFilesystem> filesystem;
AbsPath path;
tl::unique_ptr<ISource> fsSource;
uint64_t cachedSize = uint64_t(-1);
//tl::unique_ptr<zip_source> zipSource;
tl::unique_ptr<zip> zipArchive;
};
protected:
struct File;
struct Folder;
struct EntryIndex;
ZipPack(tl::unique_ref<IMapSource> zipFileSource, ZipReader zipReader);
ZipPack(tl::lent_ref<const IFilesystem> filesystem, AbsPathView zipFileLocation, ZipReader zipReader);
ZipPack(tl::shared_ptr<ZipData> data);
OpenMapSourceResult openRawMapSource(const File& file, MapView mapView) const;
OpenSourceResult openRawSource(const File& file) const;
//OpenMapSourceResult openRawMapSource(const File& file, MapView mapView) const;
//OpenSourceResult openRawSource(const File& file) const;
OpenSourceResult openZipSource(const File& file) const;
OpenMapSourceResult openZipMapSource(const File& file) const;
OpenStreamSourceResult openZipStreamSource(const File& file) const;
OpenSourceResult _openSource(const File& file) const;
OpenMapSourceResult _openMapSource(const File& file) const;
OpenStreamSourceResult _openStreamSource(const File& file) const;
uint16_t getOrAddFolder(AbsPathView folderPath, tl::vector<tl::vector<EntryIndex>>& io_childrenIndices);
void createEntries(ZipReader zipReader);
void createEntries(zip& archive);
struct File
{
uint16_t zipIndex = 0; //index inside the zip archive
tl::string name;
uint8_t isCompressed : 1;
uint8_t uncompressedSizeH = 0;
uint8_t compressedSizeH = 0;
uint8_t dataOffsetH = 0;
uint16_t parentIndex = 0;
uint32_t lastModTimePoint = 0;
uint32_t uncompressedSizeL = 0;
uint32_t compressedSizeL = 0;
uint32_t dataOffsetL = 0;
};
static_assert(sizeof(File) <= sizeof(tl::string) + 40, "Check your sizes!!!");
struct Folder
{
uint16_t zipIndex = 0; //index inside the zip archive
tl::string name;
uint16_t parentIndex = 0;
uint16_t childrenStartIndex = 0; // this is an index inside m_childrenIndices
uint16_t childrenCount = 0;
uint32_t lastModTimePoint = 0;
};
static_assert(sizeof(Folder) <= sizeof(tl::string) + 20, "Check your sizes!!!");
@@ -99,12 +126,8 @@ protected:
static_assert(sizeof(EntryIndex) == 4, "Check your sizes!!!");
private:
tl::lent_ptr<const IFilesystem> m_filesystem;
mutable std::mutex m_zipMapSourceLock;
tl::unique_ptr<IMapSource> m_zipMapSource;
AbsPath m_zipFileLocation;
tl::shared_ptr<ZipData> m_zipData;
tl::string m_encryptionKey;
uint32_t m_encryptionRounds = 32;
include/fs/zip/ZipReader.h
-63
View File
@@ -1,63 +0,0 @@
#pragma once
#include "ZipBase.h"
#include "tl/memory_buffer.h"
#include "tl/result.h"
#include "tl/vector.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class ISource;
//////////////////////////////////////////////////////////////////////////
class ZipReader : public ZipBase
{
public:
enum class ErrorCode : uint8_t
{
InvalidStream,
BadSignature,
BadOffset,
CorruptedFile,
};
typedef tl::error<ErrorCode> Error;
typedef tl::result<ZipReader, Error> CreateResult;
static CreateResult create(ISource& source);
struct Entry
{
tl::string name;
CompressionMethod compressionMethod = CompressionMethod::Store;
GeneralBitFlags generalBitFlags;
uint64_t uncompressedSize = 0;
uint64_t compressedSize = 0;
uint32_t crc32 = 0;
uint64_t localHeaderOffset = 0;
uint64_t dataOffset = 0;
time_t lastModTimePoint = 0;
tl::memory_buffer comment;
};
size_t getEntryCount() const;
const Entry& getEntry(size_t index) const;
typedef tl::result<void, Error> SeekResult;
SeekResult seekSourceToEntryData(const Entry& entry, ISource& o_source) const;
private:
explicit ZipReader(tl::vector<Entry> entries);
typedef tl::result<tl::pair<uint64_t, uint64_t>, Error> CentralDirectoryResult;
static CentralDirectoryResult findCentralDirectoryBounds(ISource& source);
tl::vector<Entry> m_entries;
};
//////////////////////////////////////////////////////////////////////////
}
include/fs/zip/ZipUtils.h
+2 -2
View File
@@ -13,7 +13,7 @@ FS_API UnzipResult unzipSource(IFilesystem& dstFilesystem, AbsPathView filePath,
FS_API UnzipResult unzipFile(IFilesystem& dstFilesystem, AbsPathView filePath, tl::lent_ref<const IFilesystem> filesystem, AbsPathView srcFilePath);
typedef tl::result<void, Error> ZipResult;
FS_API ZipResult zipToSink(ISink& sink, tl::lent_ref<IFilesystem> filesystem, AbsPathView path, uint8_t compressionLevel, size_t fileDataAlignment = 0);
FS_API ZipResult zipToFile(IFilesystem& dstFilesystem, AbsPathView dstFilePath, tl::lent_ref<IFilesystem> filesystem, AbsPathView path, uint8_t compressionLevel, size_t fileDataAlignment = 0);
FS_API ZipResult zipToSink(ISink& sink, tl::lent_ref<IFilesystem> filesystem, AbsPathView path, uint8_t compressionLevel);
FS_API ZipResult zipToFile(IFilesystem& dstFilesystem, AbsPathView dstFilePath, tl::lent_ref<IFilesystem> filesystem, AbsPathView path, uint8_t compressionLevel);
}
include/fs/zip/ZipWriter.h
+31 -43
View File
@@ -1,77 +1,65 @@
#pragma once
#include "ZipBase.h"
#include "fs/Api.h"
#include <tl/memory_buffer.h>
#include <tl/span.h>
#include <tl/vector.h>
#include <tl/result.h>
#include <tl/optional.h>
#include "fs/zip/ZipPack.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class IStreamSource;
class ISource;
class IStreamSink;
class ISink;
//////////////////////////////////////////////////////////////////////////
class FS_API ZipWriter : public ZipBase
class FS_API ZipWriter
{
public:
struct DataWriterPayload
{
CompressionMethod compressionMethod;
uint32_t uncompressedCRC32 = 0; //mandatory!!!
uint64_t uncompressedSize = 0; //optional, will be set to the written size if == 0
};
//0 - no compression (Store)
//9 - max compression (Deflate)
using Compression = uint8_t;
using DataWriterError = tl::generic_error;
using DataWriterResult = tl::result<DataWriterPayload, DataWriterError>;
ZipWriter(ISink& sink);
~ZipWriter();
using DataWriter = tl::function<DataWriterResult(IStreamSink& sink)>;
ZipWriter(ISink& sink, size_t fileDataAlignment);
~ZipWriter() = default;
tl::result<void> addFile(tl::string name, const DataWriter& dataWriter);
tl::result<void> addFile(tl::string name, tl::optional<time_t> lastModificationTime, const DataWriter& dataWriter);
tl::result<void> addBuffer(tl::string name, Compression compression, tl::span<const uint8_t> buffer);
tl::result<void> addBuffer(tl::string name, time_t lastModificationTime, Compression compression, tl::span<const uint8_t> buffer);
tl::result<void> addBuffer(tl::string name,
tl::span<const uint8_t> comment,
tl::span<const uint8_t> extraField,
time_t lastModificationTime,
Compression compression,
tl::span<const uint8_t> buffer);
tl::result<void> addFile(tl::string name, Compression compression, ISource& source);
tl::result<void> addFile(tl::string name, time_t lastModificationTime, Compression compression, ISource& source);
tl::result<void> addFile(tl::string name,
tl::span<const uint8_t> comment,
tl::span<const uint8_t> extraField,
GeneralBitFlags generalBitFlags,
tl::optional<time_t> lastModificationTime,
const DataWriter& dataWriter);
tl::span<const uint8_t> comment,
tl::span<const uint8_t> extraField,
time_t lastModificationTime,
Compression compression,
ISource& source);
void finish();
private:
uint64_t computePadding(uint64_t offset, const LocalFileHeader& header) const;
tl::result<uint64_t> _addSource(tl::string name,
tl::span<const uint8_t> comment,
tl::span<const uint8_t> extraField,
time_t lastModificationTime,
Compression compression,
zip_source* source);
ISink& m_sink;
size_t m_fileDataAlignment = 0;
struct CentralDirectoryRecord
{
uint64_t compressedSize = 0;
uint64_t uncompressedSize = 0;
uint64_t localHeaderOffset = 0;
CentralDirectoryFileHeader header;
tl::string name;
tl::memory_buffer comment;
tl::memory_buffer extraField;
bool zip64 = false;
};
tl::vector<CentralDirectoryRecord> m_records;
tl::memory_buffer m_buffer;
tl::unique_ptr<zip> m_zipArchive;
};
//////////////////////////////////////////////////////////////////////////
}