This commit is contained in:
jeanlemotan
2024-07-02 18:12:23 +02:00
commit b344afa9fe
89 changed files with 10568 additions and 0 deletions
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include "fs/AbsPath.h"
#include "fs/zip/ZipWriter.h"
#include "tl/ptr.h"
namespace fs
{
class IFilesystem;
//////////////////////////////////////////////////////////////////////////
class FS_API DeflateFileWriter
{
public:
//compression level: 0 to 9
DeflateFileWriter(AbsPath i_filePath, tl::lent_ref<IFilesystem> i_filesystem, uint8_t i_compressionLevel);
DeflateFileWriter(const DeflateFileWriter&) = delete;
DeflateFileWriter& operator=(const DeflateFileWriter&) = delete;
DeflateFileWriter(DeflateFileWriter&&) = default;
DeflateFileWriter& operator=(DeflateFileWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& i_sink);
private:
AbsPath m_filePath;
tl::lent_ref<IFilesystem> m_filesystem;
uint8_t m_compressionLevel = 9;
};
//////////////////////////////////////////////////////////////////////////
}
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include "fs/IStreamSource.h"
#include "fs/zip/ZipWriter.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class ISource;
//////////////////////////////////////////////////////////////////////////
class FS_API DeflateSinkWriter
{
public:
//compression level: 0 to 9
DeflateSinkWriter(IStreamSource& source, uint8_t compressionLevel);
DeflateSinkWriter(tl::span<const uint8_t> data, uint8_t compressionLevel);
DeflateSinkWriter(const DeflateSinkWriter&) = delete;
DeflateSinkWriter& operator=(const DeflateSinkWriter&) = delete;
DeflateSinkWriter(DeflateSinkWriter&&) = default;
DeflateSinkWriter& operator=(DeflateSinkWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& sink);
private:
IStreamSource* m_source = nullptr;
tl::span<const uint8_t> m_data;
uint8_t m_compressionLevel = 9;
};
//////////////////////////////////////////////////////////////////////////
}
+35
View File
@@ -0,0 +1,35 @@
#pragma once
#include "fs/AbsPath.h"
#include "fs/zip/ZipWriter.h"
#include "tl/ptr.h"
namespace fs
{
class IFilesystem;
//////////////////////////////////////////////////////////////////////////
class FS_API StoreFileWriter
{
public:
StoreFileWriter(AbsPath i_filePath, tl::lent_ref<IFilesystem> i_filesystem);
StoreFileWriter(const StoreFileWriter&) = delete;
StoreFileWriter& operator=(const StoreFileWriter&) = delete;
StoreFileWriter(StoreFileWriter&&) = default;
StoreFileWriter& operator=(StoreFileWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& i_sink);
private:
AbsPath m_filePath;
tl::lent_ref<IFilesystem> m_filesystem;
};
//////////////////////////////////////////////////////////////////////////
}
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "fs/zip/ZipWriter.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class ISource;
//////////////////////////////////////////////////////////////////////////
class FS_API StoreSinkWriter
{
public:
explicit StoreSinkWriter(ISource& source);
StoreSinkWriter(const StoreSinkWriter&) = delete;
StoreSinkWriter& operator=(const StoreSinkWriter&) = delete;
StoreSinkWriter(StoreSinkWriter&&) = default;
StoreSinkWriter& operator=(StoreSinkWriter&&) = default;
ZipWriter::DataWriterResult operator()(IStreamSink& sink);
private:
tl::reference_wrapper<ISource> m_inSource;
};
//////////////////////////////////////////////////////////////////////////
}
+158
View File
@@ -0,0 +1,158 @@
#pragma once
#include "tl/flag_set.h"
#include "fs/Api.h"
namespace fs
{
class FS_API ZipBase
{
public:
enum class GeneralBitFlag : uint16_t
{
Encrypted = 1 << 0,
CompressionMethodSpecific1 = 1 << 1,
CompressionMethodSpecific2 = 1 << 2,
DataDescriptorNeeded = 1 << 3,
StrongEncryption = 1 << 6,
LanguageEncodingFlag = 1 << 11,
LocalHeaderFieldsMasked = 1 << 13,
};
typedef tl::flag_set<GeneralBitFlag> GeneralBitFlags;
enum class CompressionMethod : uint16_t
{
Store = 0,
Shrunk = 1,
Reduced1 = 2,
Reduced2 = 3,
Reduced3 = 4,
Reduced4 = 5,
Implode = 6,
//Reserved = 7,
Deflate = 8,
Deflate64 = 9,
PKWareImplode = 10,
//Reserved = 11,
BZip2 = 12,
//Reserved = 13,
LZMA = 14,
//Reserved = 15,
//Reserved = 16,
//Reserved = 17,
Terse = 18,
LZ77 = 19,
LZ4 = 20, // non-standard
ZStd = 21, // non-standard
WavPack = 97,
PPMd = 98,
};
protected:
//////////////////////////////////////////////////////////////////////////
//These structs follow pretty closely the https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT standard
#pragma pack(push, 1)
struct LocalFileHeader
{
static constexpr uint32_t k_signature = 0x04034b50u;
uint32_t signature = k_signature;
uint16_t versionNeededToExtract = 0;
uint16_t generalBitFlag = 0;
uint16_t compressionMethod = 0;
uint16_t lastModFileTime = 0;
uint16_t lastModFileDate = 0;
uint32_t crc32 = 0;
uint32_t compressedSize = 0;
uint32_t uncompressedSize = 0;
uint16_t filenameLength = 0;
uint16_t extraFieldLength = 0;
};
struct DataDescriptor
{
uint32_t crc32 = 0;
uint32_t compressedSize = 0;
uint32_t uncompressedSize = 0;
};
struct CentralDirectoryFileHeader
{
static constexpr uint32_t k_signature = 0x02014b50u;
uint32_t signature = k_signature;
uint16_t versionMadeBy = 0;
uint16_t versionNeededToExtract = 0;
uint16_t generalBitFlag = 0;
uint16_t compressionMethod = 0;
uint16_t lastModFileTime = 0;
uint16_t lastModFileDate = 0;
uint32_t crc32 = 0;
uint32_t compressedSize = 0;
uint32_t uncompressedSize = 0;
uint16_t filenameLength = 0;
uint16_t extraFieldLength = 0;
uint16_t fileCommentLength = 0;
uint16_t diskNumberStart = 0;
uint16_t internalFileAttributes = 0;
uint32_t externalFileAttributes = 0;
uint32_t localHeaderOffset = 0;
};
struct EndOfCentralDirectoryRecord
{
static constexpr uint32_t k_signature = 0x06054b50u;
uint32_t signature = k_signature;
uint16_t diskNumber = 0;
uint16_t centralDirectoryStartDiskNumber = 0;
uint16_t centralDirectoryRecordCountOnThisDisk = 0;
uint16_t centralDirectoryRecordCount = 0;
uint32_t centralDirectorySize = 0;
uint32_t centralDirectoryOffset = 0;
uint16_t zipFileCommentLength = 0;
};
struct ExtendedInformationExtraField64
{
static constexpr uint32_t k_tag = 0x0001;
uint16_t tag = k_tag;
uint16_t size = 0;
uint64_t originalSize = 0;
uint64_t compressedSize = 0;
uint64_t localHeaderOffset = 0;
uint32_t diskNumber = 0;
};
struct EndOfCentralDirectoryRecord64
{
static constexpr uint32_t k_signature = 0x06064b50u;
uint32_t signature = k_signature;
uint64_t endOfCentralDirectorySize = 0;
uint16_t versionMadeBy = 0;
uint16_t versionNeededToExtract = 0;
uint32_t diskNumber = 0;
uint32_t centralDirectoryStartDiskNumber = 0;
uint64_t centralDirectoryRecordCountOnThisDisk = 0;
uint64_t centralDirectoryRecordCount = 0;
uint64_t centralDirectorySize = 0;
uint64_t centralDirectoryOffset = 0;
};
struct EndOfCentralDirectoryLocator64
{
static constexpr uint32_t k_signature = 0x07064b50u;
uint32_t signature = k_signature;
uint32_t centralDirectoryStartDiskNumber = 0;
uint64_t endOfCentralDirectoryRecordOffset = 0;
uint32_t diskCount = 0;
};
#pragma pack(pop)
static void computeMsDosTime(time_t time, uint16_t& o_date, uint16_t& o_time);
static CentralDirectoryFileHeader computeCentralDirectoryFileHeader(const LocalFileHeader& header, bool zip64);
};
}
+121
View File
@@ -0,0 +1,121 @@
#pragma once
#include "tl/result.h"
#include "tl/string.h"
#include "tl/vector_map.h"
#include "fs/IPack.h"
#include "fs/AbsPath.h"
#include "fs/zip/ZipReader.h"
#include "fs/Api.h"
namespace fs
{
class MapSourceView;
class FS_API ZipPack final : virtual public IPack
{
public:
ZipPack() = default;
~ZipPack() override = default;
typedef tl::result<tl::unique_ref<ZipPack>, Error> CreateResult;
static CreateResult create(tl::unique_ref<IMapSource> zipFileSource);
static CreateResult create(tl::lent_ref<const IFilesystem> filesystem, AbsPath zipFileLocation);
virtual void setEncryptionData(const tl::string& key, uint32_t rounds);
OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override;
OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override;
IsFileResult isFile(const AbsPath& path) const override;
IsFolderResult isFolder(const AbsPath& path) const override;
ExistsResult exists(const AbsPath& path) const override;
GetStatResult getStat(const AbsPath& path) const override;
cppcoro::generator<EnumerateEntry> enumerate(const AbsPath& path) const override;
cppcoro::generator<EnumerateEntry> enumerateRecursively(const AbsPath& path) const override;
tl::result<AbsPath, Error> convertToNativePath(const AbsPath& path) const override;
protected:
struct File;
struct Folder;
struct EntryIndex;
ZipPack(tl::unique_ref<IMapSource> zipFileSource, ZipReader zipReader);
ZipPack(tl::lent_ref<const IFilesystem> filesystem, AbsPath zipFileLocation, ZipReader zipReader);
OpenMapSourceResult openRawMapSource(const File& file, MapView mapView) const;
OpenSourceResult openRawSource(const File& file) const;
OpenSourceResult openZipSource(const File& file) const;
OpenMapSourceResult openZipMapSource(const File& file) const;
OpenStreamSourceResult openZipStreamSource(const File& file) const;
uint16_t getOrAddFolder(AbsPath folderPath, tl::vector<tl::vector<EntryIndex>>& io_childrenIndices);
void createEntries(ZipReader zipReader);
struct File
{
tl::string name;
uint8_t isCompressed : 1;
uint8_t uncompressedSizeH = 0;
uint8_t compressedSizeH = 0;
uint8_t dataOffsetH = 0;
uint16_t parentIndex = 0;
uint32_t lastModTimePoint = 0;
uint32_t uncompressedSizeL = 0;
uint32_t compressedSizeL = 0;
uint32_t dataOffsetL = 0;
};
static_assert(sizeof(File) <= sizeof(tl::string) + 40, "Check your sizes!!!");
struct Folder
{
tl::string name;
uint16_t parentIndex = 0;
uint16_t childrenStartIndex = 0; // this is an index inside m_childrenIndices
uint16_t childrenCount = 0;
uint32_t lastModTimePoint = 0;
};
static_assert(sizeof(Folder) <= sizeof(tl::string) + 20, "Check your sizes!!!");
struct EntryIndex
{
EntryIndex() = default;
EntryIndex(bool isFolder, uint16_t index)
: isFolder(isFolder)
, index(index)
{
}
uint32_t isFolder : 1;
uint32_t index : 16;
};
static_assert(sizeof(EntryIndex) == 4, "Check your sizes!!!");
private:
tl::lent_ptr<const IFilesystem> m_filesystem;
mutable std::mutex m_zipMapSourceLock;
tl::unique_ptr<IMapSource> m_zipMapSource;
AbsPath m_zipFileLocation;
tl::string m_encryptionKey;
uint32_t m_encryptionRounds = 32;
tl::vector_map<uint64_t, EntryIndex> m_pathToIndex;
tl::vector<File> m_files;
tl::vector<Folder> m_folders;
tl::vector<EntryIndex> m_childrenIndices; // these are indices inside the m_files or m_folders
void buildUpPath(RelPath& path, const Folder& folder, uint32_t toIndex) const;
void buildUpPath(RelPath& path, const File& file, uint32_t toIndex) const;
};
}
+63
View File
@@ -0,0 +1,63 @@
#pragma once
#include "ZipBase.h"
#include "tl/memory_buffer.h"
#include "tl/result.h"
#include "tl/vector.h"
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class ISource;
//////////////////////////////////////////////////////////////////////////
class ZipReader : public ZipBase
{
public:
enum class ErrorCode : uint8_t
{
InvalidStream,
BadSignature,
BadOffset,
CorruptedFile,
};
typedef tl::error<ErrorCode> Error;
typedef tl::result<ZipReader, Error> CreateResult;
static CreateResult create(ISource& source);
struct Entry
{
tl::string name;
CompressionMethod compressionMethod = CompressionMethod::Store;
GeneralBitFlags generalBitFlags;
uint64_t uncompressedSize = 0;
uint64_t compressedSize = 0;
uint32_t crc32 = 0;
uint64_t localHeaderOffset = 0;
uint64_t dataOffset = 0;
time_t lastModTimePoint = 0;
tl::memory_buffer comment;
};
size_t getEntryCount() const;
const Entry& getEntry(size_t index) const;
typedef tl::result<void, Error> SeekResult;
SeekResult seekSourceToEntryData(const Entry& entry, ISource& o_source) const;
private:
explicit ZipReader(tl::vector<Entry> entries);
typedef tl::result<tl::pair<uint64_t, uint64_t>, Error> CentralDirectoryResult;
static CentralDirectoryResult findCentralDirectoryBounds(ISource& source);
tl::vector<Entry> m_entries;
};
//////////////////////////////////////////////////////////////////////////
}
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include "fs/Error.h"
#include "fs/Api.h"
#include "fs/IFilesystem.h"
namespace fs
{
class ISink;
typedef tl::result<void, Error> UnzipResult;
FS_API UnzipResult unzipSource(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::unique_ref<IMapSource> source);
FS_API UnzipResult unzipFile(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::lent_ref<const IFilesystem> filesystem, AbsPath srcFilePath);
typedef tl::result<void, Error> ZipResult;
FS_API ZipResult zipToSink(ISink& sink, tl::lent_ref<IFilesystem> filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment = 0);
FS_API ZipResult zipToFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, tl::lent_ref<IFilesystem> filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment = 0);
}
+77
View File
@@ -0,0 +1,77 @@
#pragma once
#include "ZipBase.h"
#include "fs/Api.h"
#include <tl/memory_buffer.h>
#include <tl/span.h>
#include <tl/vector.h>
#include <tl/result.h>
#include <tl/optional.h>
namespace fs
{
//////////////////////////////////////////////////////////////////////////
class IStreamSource;
class IStreamSink;
class ISink;
//////////////////////////////////////////////////////////////////////////
class FS_API ZipWriter : public ZipBase
{
public:
struct DataWriterPayload
{
CompressionMethod compressionMethod;
uint32_t uncompressedCRC32 = 0; //mandatory!!!
uint64_t uncompressedSize = 0; //optional, will be set to the written size if == 0
};
using DataWriterError = tl::generic_error;
using DataWriterResult = tl::result<DataWriterPayload, DataWriterError>;
using DataWriter = tl::function<DataWriterResult(IStreamSink& sink)>;
ZipWriter(ISink& sink, size_t fileDataAlignment);
~ZipWriter() = default;
tl::result<void> addFile(tl::string name, const DataWriter& dataWriter);
tl::result<void> addFile(tl::string name, tl::optional<time_t> lastModificationTime, const DataWriter& dataWriter);
tl::result<void> addFile(tl::string name,
tl::span<const uint8_t> comment,
tl::span<const uint8_t> extraField,
GeneralBitFlags generalBitFlags,
tl::optional<time_t> lastModificationTime,
const DataWriter& dataWriter);
void finish();
private:
uint64_t computePadding(uint64_t offset, const LocalFileHeader& header) const;
ISink& m_sink;
size_t m_fileDataAlignment = 0;
struct CentralDirectoryRecord
{
uint64_t compressedSize = 0;
uint64_t uncompressedSize = 0;
uint64_t localHeaderOffset = 0;
CentralDirectoryFileHeader header;
tl::string name;
tl::memory_buffer comment;
tl::memory_buffer extraField;
bool zip64 = false;
};
tl::vector<CentralDirectoryRecord> m_records;
tl::memory_buffer m_buffer;
};
//////////////////////////////////////////////////////////////////////////
}