commit b344afa9fe2406e6a8e4663a2aec84315e41595a Author: jeanlemotan Date: Tue Jul 2 18:12:23 2024 +0200 First diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..b5c81ba --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,85 @@ +cmake_minimum_required(VERSION 3.16.5) + +project(FS) + +include(${PROJECT_SOURCE_DIR}/../build-utils/cmake/compiler_settings.cmake) + +if (MSVC) + add_definitions(/bigobj) + set(LIB_FOLDER "vs2022") +elseif (ANDROID) + set(LIB_FOLDER "Android") +elseif (APPLE) + set(LIB_FOLDER "macos") +endif() + +if(NOT TARGET TL) + add_subdirectory("${PROJECT_SOURCE_DIR}/../TL" "./TL") +endif() + +if(NOT TARGET EASTL) + set(EASTL_DIR "${PROJECT_SOURCE_DIR}/../EASTL") + set(EASTL_LIB_DIR "${EASTL_DIR}/release/lib/${LIB_FOLDER}") + add_library(EASTL STATIC IMPORTED) + list(APPEND EASTL_INCLUDE_DIRS "${EASTL_DIR}/include" + "${EASTL_DIR}/test/packages/EAAssert/include" + "${EASTL_DIR}/test/packages/EABase/include/Common" + "${EASTL_DIR}/test/packages/EAMain/include" + "${EASTL_DIR}/test/packages/EAStdC/include" + "${EASTL_DIR}/test/packages/EATest/include" + "${EASTL_DIR}/test/packages/EAThread/include") + list(APPEND LIB_INCLUDE_DIRS "dir2") + set_target_properties(EASTL PROPERTIES + IMPORTED_LOCATION "${EASTL_LIB_DIR}/Release/${LIB_PREFIX}EASTL${LIB_SUFFIX}" + IMPORTED_LOCATION_DEBUG "${EASTL_LIB_DIR}/Debug/${LIB_PREFIX}EASTL${LIB_SUFFIX}" + INTERFACE_INCLUDE_DIRECTORIES "${EASTL_INCLUDE_DIRS}") +endif() + +if(NOT TARGET zlib) + set(ZLIB_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/../zlib/release/include") + set(ZLIB_LIB_DIR "${PROJECT_SOURCE_DIR}/../zlib/release/lib/${LIB_FOLDER}") + add_library(zlib STATIC IMPORTED) + if (MSVC) + set_target_properties(zlib PROPERTIES + IMPORTED_LOCATION "${ZLIB_LIB_DIR}/zlibstatic-ng.lib" + INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) + else() + set_target_properties(zlib PROPERTIES + IMPORTED_LOCATION "${ZLIB_LIB_DIR}/libz-ng.a" + INTERFACE_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) + endif() +endif() + +file(GLOB_RECURSE SRC "src/*.cpp") +file(GLOB_RECURSE HEADERS "include/*.h" "include/*.inl") + +foreach(_source IN ITEMS ${HEADERS}) +get_filename_component(_source_path "${_source}" PATH) +file(RELATIVE_PATH _source_path_rel "${PROJECT_SOURCE_DIR}" "${_source_path}") +string(REPLACE "/" "\\" _group_path "${_source_path_rel}") +source_group("${_group_path}" FILES "${_source}") +endforeach() + +foreach(_source IN ITEMS ${SRC}) +get_filename_component(_source_path "${_source}" PATH) +file(RELATIVE_PATH _source_path_rel "${PROJECT_SOURCE_DIR}" "${_source_path}") +string(REPLACE "/" "\\" _group_path "${_source_path_rel}") +source_group("${_group_path}" FILES "${_source}") +endforeach() + +add_library(FS STATIC ${SRC} ${HEADERS}) +target_include_directories(FS PUBLIC "include") +target_include_directories(FS PRIVATE "src") +target_include_directories(FS PUBLIC "${PROJECT_SOURCE_DIR}/../Logger/include") +target_include_directories(FS PUBLIC "${PROJECT_SOURCE_DIR}/../cppcoro/include") +target_include_directories(FS PUBLIC "${PROJECT_SOURCE_DIR}/../fast_io/include") + +target_link_libraries(FS TL) +target_link_libraries(FS zlib) +target_link_libraries(FS EASTL) + +target_precompile_headers(FS PRIVATE "$<$:${CMAKE_CURRENT_SOURCE_DIR}/src/StdAfx.h>") + +#if(BUILD_SHARED_LIBS) +# target_compile_definitions(FS PUBLIC FS_BUILD_SHARED_LIB) +#endif() \ No newline at end of file diff --git a/include/fs/AbsPath.h b/include/fs/AbsPath.h new file mode 100644 index 0000000..a84d6c4 --- /dev/null +++ b/include/fs/AbsPath.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include "tl/string.h" +#include "tl/path_system.h" +#include "tl/rel_path.h" +#include "tl/abs_path.h" + +namespace fs +{ + +// custom windows path system +template +struct WindowsSystemCustom : public tl::detail::path_system::base_path_system +{ + template + static T format_absolute(const tl::vector& members); + static void parse_absolute(tl::vector& o_elements, const char* path, size_t size); + static bool validate_abs_path(const tl::vector& members); + static bool match(const char* path, size_t size); +}; +TL_DECLARE_STRING_VECTOR(WindowsParseSeparators, "\\", "/"); +TL_DECLARE_STRING_LITERAL(WindowsFormatSeparator, "\\"); +using WindowsSystem = WindowsSystemCustom; + +// posix path system +TL_DECLARE_STRING_LITERAL(PosixRootTag, "/"); +TL_DECLARE_STRING_VECTOR(PosixParseSeparators, "/"); +TL_DECLARE_STRING_LITERAL(PosixFormatSeparator, "/"); +using PosixSystem = tl::simple_path_system; + +// mac path system +TL_DECLARE_STRING_LITERAL(UncRootTag, "\\\\"); +TL_DECLARE_STRING_VECTOR(UncParseSeparators, "\\", "/"); +TL_DECLARE_STRING_LITERAL(UncFormatSeparator, "\\"); +using UncSystem = tl::simple_path_system; + +using AbsPath = tl::abs_path >; +using RelPath = AbsPath::rel_path_type; + +} + +#include "fs/AbsPath.inl" + +template <> +struct std::formatter +{ + constexpr auto parse(format_parse_context& ctx) noexcept { return ctx.begin(); } + auto format(const fs::RelPath& p, std::format_context& ctx) const + { + return format_to(ctx.out(), "{}", p.get_as()); + } +}; diff --git a/include/fs/AbsPath.inl b/include/fs/AbsPath.inl new file mode 100644 index 0000000..62636b1 --- /dev/null +++ b/include/fs/AbsPath.inl @@ -0,0 +1,51 @@ +#pragma once + +#include + +namespace fs +{ +namespace detail +{ + TL_DECLARE_STRING_LITERAL(EmptyRootTag, ""); +} + + // windows path system + template + template + T WindowsSystemCustom::format_absolute(const tl::vector& members) + { + return tl::simple_path_system::template format_absolute(members); + } + + template + void WindowsSystemCustom::parse_absolute(tl::vector& o_elements, const char* path, size_t size) + { + // On windows, the drive letter case is undefined. To avoid issues (hashing, etc), we always convert the drive letter to uppercase + tl::simple_path_system::parse_absolute(o_elements, path, size); + if (!o_elements.empty()) + { + tl::string& str = o_elements.front(); + if (str.length() == 2 && str[1] == ':') + { + char strUp[2]; + strUp[0] = tl::ascii::toupper(str[0]); + strUp[1] = ':'; + str = tl::string(strUp, 2); + } + } + } + + template + bool WindowsSystemCustom::match(const char* path, size_t size) + { + return size >= 2 && std::isalpha(path[0], std::locale()) && path[1] == ':'; + } + + template + bool WindowsSystemCustom::validate_abs_path(const tl::vector& members) + { + return !members.empty() && match(members[0].c_str(), members[0].size()); + } + +////////////////////////////////////////////////////////////////////////// +} diff --git a/include/fs/Api.h b/include/fs/Api.h new file mode 100644 index 0000000..c929477 --- /dev/null +++ b/include/fs/Api.h @@ -0,0 +1,23 @@ +#pragma once + +#include "tl/platform.h" + +#if defined FS_BUILD_SHARED_LIB + + #if defined TL_TOOLCHAIN_MSC + # define FS_API __declspec(dllexport) + #else + # define FS_API + #endif + +#elif defined FS_USE_SHARED_LIB + + #if defined TL_TOOLCHAIN_MSC + # define FS_API __declspec(dllimport) + #else + # define FS_API + #endif // + +#else +# define FS_API +#endif diff --git a/include/fs/BufferedFileSource.h b/include/fs/BufferedFileSource.h new file mode 100644 index 0000000..72c9744 --- /dev/null +++ b/include/fs/BufferedFileSource.h @@ -0,0 +1,14 @@ +#pragma once + +#include "fs/BufferedSource.h" +#include "fs/FileSource.h" + +namespace fs +{ +/** + * Buffered version of the FileSource ideal for doing random access to file. + * for further information please take a look to BufferedSource. + */ +using BufferedFileSource = BufferedSource; + +} diff --git a/include/fs/BufferedSource.h b/include/fs/BufferedSource.h new file mode 100644 index 0000000..757c415 --- /dev/null +++ b/include/fs/BufferedSource.h @@ -0,0 +1,47 @@ +#pragma once + +#include "fs/ISource.h" +#include "tl/memory_buffer.h" + +namespace fs +{ +/** + * ISource interface implementation where the access to the underneath source + * is done in buffered mode. This class must be used when there a lot of tiny + * random access to the source. For further information please check ISource. + */ +template +class BufferedSource final : public ISource +{ +public: + explicit BufferedSource(SourceType source, size_t bufferSize = 4096u); + + BufferedSource(BufferedSource&& other) = default; + BufferedSource& operator=(BufferedSource&& other) = default; + BufferedSource(const BufferedSource& other) = delete; + BufferedSource& operator=(const BufferedSource& other) = delete; + + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + + uint64_t tell() const override; + + uint64_t getSize() const override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; +private: + SourceType m_source; + uint64_t m_bufferStartOffset = 0; + uint64_t m_bufferOffset = 0; + bool m_endOfSource = false; + tl::memory_buffer m_buffer; + mutable tl::optional m_optLastError; +}; + +} diff --git a/include/fs/BufferedSource.inl b/include/fs/BufferedSource.inl new file mode 100644 index 0000000..0a32743 --- /dev/null +++ b/include/fs/BufferedSource.inl @@ -0,0 +1,150 @@ + +#include "fs/BufferedSource.h" +#include + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +BufferedSource::BufferedSource(SourceType source, size_t bufferSize) + : m_source(std::move(source)) +{ + if (bufferSize < 8) + bufferSize = 8; + + m_buffer.reserve(bufferSize); +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +tl::optional BufferedSource::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +size_t BufferedSource::read(tl::span data) +{ + if (data.empty() || m_endOfSource) + return 0; + + auto bufferPtr = data.data(); + size_t totalSize = 0; + size_t pendingSize = data.size(); + + while (pendingSize) + { + if (m_buffer.empty() || m_bufferOffset >= m_buffer.size()) + { + m_buffer.resize_uninitialized(m_buffer.capacity()); + m_bufferStartOffset = m_source.tell(); + m_bufferOffset = 0; + const size_t readSize = m_source.read(m_buffer); + m_buffer.resize_uninitialized(readSize); + if (readSize == 0) + { + m_optLastError = m_source.getLastError(); + m_endOfSource = !m_optLastError.has_value(); //no error? then EOS + return totalSize; + } + } + + TL_ASSERT(m_buffer.size()); + TL_ASSERT(m_buffer.size() > m_bufferOffset); + const size_t frameSize = tl::min(m_buffer.size() - m_bufferOffset, pendingSize); + TL_ASSERT(frameSize); + + memcpy(bufferPtr, m_buffer.data() + m_bufferOffset, frameSize); + bufferPtr += frameSize; + + TL_ASSERT(data.size() >= frameSize); + pendingSize -= frameSize; + + totalSize += frameSize; + TL_ASSERT(totalSize <= data.size()); + + m_bufferOffset += frameSize; + } + + m_endOfSource = tell() >= getSize(); + + return totalSize; +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +void BufferedSource::seekBeg(uint64_t offset) +{ + offset = tl::min(offset, m_source.getSize()); + if (offset < m_bufferStartOffset || offset >= m_bufferStartOffset + m_buffer.size()) + { + m_source.seekBeg(offset); + m_optLastError = m_source.getLastError(); + m_buffer.clear(); + m_bufferOffset = 0; + m_bufferStartOffset = m_source.tell(); + } + else + m_bufferOffset = offset - m_bufferStartOffset; + + m_endOfSource = tell() >= getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +void BufferedSource::seekRel(int64_t offset) +{ + if (!offset) + return; + + if (offset != 0) + { + int64_t newOffset = tell(); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, getSize()))); + } +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +uint64_t BufferedSource::tell() const +{ + return m_bufferStartOffset + m_bufferOffset; +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +uint64_t BufferedSource::getSize() const +{ + return m_source.getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +bool BufferedSource::isEOS() const +{ + return m_endOfSource; +} + +////////////////////////////////////////////////////////////////////////// + +template< typename SourceType> +size_t BufferedSource::getPreferredBufferSize() const +{ + return m_buffer.size(); +} + +} + diff --git a/include/fs/CRCStream.h b/include/fs/CRCStream.h new file mode 100644 index 0000000..2e5bd70 --- /dev/null +++ b/include/fs/CRCStream.h @@ -0,0 +1,25 @@ +#pragma once +#include "Error.h" +#include "fs/Api.h" +#include "fs/AbsPath.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +class IStreamSource; +class IFilesystem; + +FS_API uint32_t computeCRC32(uint32_t crc, tl::span data); +FS_API uint64_t computeCRC64(uint64_t crc, tl::span data); + +typedef tl::result CRC32StreamResult; +FS_API CRC32StreamResult crc32Stream(IStreamSource& source, uint32_t crc = 0, size_t bufferSize = 0); +FS_API CRC32StreamResult crc32File(const IFilesystem& filesystem, const AbsPath& filePath, uint32_t crc = 0, size_t bufferSize = 0); + +typedef tl::result CRC64StreamResult; +FS_API CRC64StreamResult crc64Stream(IStreamSource& source, uint64_t crc = 0, size_t bufferSize = 0); +FS_API CRC64StreamResult crc64File(const IFilesystem& filesystem, const AbsPath& filePath, uint64_t crc = 0, size_t bufferSize = 0); + +////////////////////////////////////////////////////////////////////////// +} diff --git a/include/fs/CopyStream.h b/include/fs/CopyStream.h new file mode 100644 index 0000000..cb18618 --- /dev/null +++ b/include/fs/CopyStream.h @@ -0,0 +1,24 @@ +#pragma once +#include "Error.h" +#include "fs/Api.h" +#include "fs/ProcessStream.h" +#include "tl/result.h" +#include "fs/AbsPath.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +class IStreamSink; +class IStreamSource; +class IFilesystem; + +using CopyStreamResult = tl::result; +FS_API CopyStreamResult copyStream(IStreamSink& sink, IStreamSource& source, size_t bufferSize = 0); +FS_API CopyStreamResult copyStream(IStreamSink& sink, IStreamSource& source, const ProcessDataCallback& dataCallback, size_t bufferSize = 0); +FS_API CopyStreamResult copyFile(IStreamSink& sink, const IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0); +FS_API CopyStreamResult copyFile(IStreamSink& sink, const IFilesystem& filesystem, const AbsPath& filePath, const ProcessDataCallback& dataCallback, size_t bufferSize = 0); +FS_API CopyStreamResult copyFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, const IFilesystem& srcFilesystem, const AbsPath& srcFilePath, size_t bufferSize = 0); + +////////////////////////////////////////////////////////////////////////// +} diff --git a/include/fs/CustomFilesystem.h b/include/fs/CustomFilesystem.h new file mode 100644 index 0000000..a6e2315 --- /dev/null +++ b/include/fs/CustomFilesystem.h @@ -0,0 +1,99 @@ +#pragma once + +#include "fs/IPack.h" +#include "fs/IFilesystem.h" +#include "fs/Error.h" +#include "fs/Mode.h" +#include "fs/Api.h" +#include "tl/vector.h" +#include "tl/identifier.h" + +namespace fs +{ + +class FS_API CustomFilesystem final : public IFilesystem +{ +public: + CustomFilesystem() = default; + CustomFilesystem(CustomFilesystem&&) = delete; + CustomFilesystem& operator=(CustomFilesystem&&) = delete; + CustomFilesystem(const CustomFilesystem&) = delete; + CustomFilesystem& operator=(const CustomFilesystem&) = delete; + + ////////////////////////////////////////////////////////////////////////// + // New API + + TL_DECLARE_INTEGRAL_ID(PackId, uint64_t, 0) + + tl::result mountFront(AbsPath mountPoint, tl::unique_ref pack); + tl::result mountBack(AbsPath mountPoint, tl::unique_ref pack); + + //unmounts all the packs for this point. It basically deletes the mount point + cppcoro::generator> unmountAll(); + + //When unmounting, the Filesystem returns the pack back to you (if found) + tl::unique_ptr unmount(PackId packId); + + //Returns the physical location of the virtual pack. + ConvertToNativePathResult convertToNativePath(const AbsPath& path) const override; + + OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override; + OpenSinkResult openSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override; + OpenStreamSinkResult openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override; + OpenMapSinkResult openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags = SinkFlags()) override; + + IsFileResult isFile(const AbsPath& path) const override; + IsFolderResult isFolder(const AbsPath& path) const override; + ExistsResult exists(const AbsPath& path) const override; + + MakeFolderResult makeFolder(const AbsPath& path) override; + + RenameResult rename(const AbsPath& path, const AbsPath& newPath) override; + + RemoveResult remove(const AbsPath& path) override; + RemoveRecursivelyResult removeRecursively(const AbsPath& path) override; + + CopyResult copy(const AbsPath& path, const AbsPath& newPath) override; + + MakeHardLinkResult makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) override; + MakeSoftLinkResult makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) override; + + SetWriteTimeResult setWriteTime(const AbsPath& path, time_t time) override; + + GetStatResult getStat(const AbsPath& path) const override; + + cppcoro::generator enumerate(const AbsPath& path) const override; + cppcoro::generator enumerateRecursively(const AbsPath& path) const override; + +private: + struct PackData + { + PackId id; + tl::unique_ref pack; + AbsPath mountPoint; + + explicit PackData(tl::unique_ref&& pack) noexcept + : pack(std::move(pack)) + {} + + PackData(PackData&& other) noexcept = default; + PackData& operator=(PackData&& other) noexcept = default; + PackData(const PackData& other) noexcept = delete; + PackData& operator=(const PackData& other) noexcept = delete; + }; + tl::vector m_packsData; + + void convertToPackPath(AbsPath& packPath, const AbsPath& path, const AbsPath& mountPoint) const; + + enum class MountPolicy + { + Front, //the pack will be mounted over packs from the same mount point. This pack will take precedence + Back //the pack will be at the bottom of existing packs from the same mount point. Its streams will be searched last + }; + tl::result mount(AbsPath mountPoint, tl::unique_ref pack, MountPolicy policy); + +}; + +} diff --git a/include/fs/DeflateStreamSink.h b/include/fs/DeflateStreamSink.h new file mode 100644 index 0000000..e1f8afc --- /dev/null +++ b/include/fs/DeflateStreamSink.h @@ -0,0 +1,54 @@ +#pragma once +#include "fs/IStreamSink.h" +#include "tl/memory_buffer.h" +#include "fs/Api.h" +#include "tl/ptr.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +class FS_API DeflateStreamSink final: public IStreamSink +{ +public: + explicit DeflateStreamSink(IStreamSink& sink); //non owning + explicit DeflateStreamSink(tl::unique_ref sink); //owning + ~DeflateStreamSink() override; + + //compression level - 0 to 9 + void setCompressionLevel(uint8_t level); + + tl::optional getLastError() const override; + size_t write(tl::span data) override; + + size_t getPreferredBufferSize() const override; + + uint64_t getSize() const override; + + //Finish is also called by the destructor + void finish(); + tl::unique_ref finishAndExtract(); + +private: + tl::unique_ptr m_ownedSink; + IStreamSink& m_sink; + + uint8_t m_level = 6; + mutable tl::optional m_optLastError; + + void initializeDeflate(); + bool flush(bool finalize); + + struct ZLib; + tl::unique_ptr m_zlib; + + tl::memory_buffer m_buffer; + size_t m_bufferOffset = 0; + uint64_t m_size = 0; + + tl::memory_buffer m_outBuffer; + bool m_isFinished = false; +}; + +////////////////////////////////////////////////////////////////////////// +} diff --git a/include/fs/DeflateStreamSource.h b/include/fs/DeflateStreamSource.h new file mode 100644 index 0000000..9306135 --- /dev/null +++ b/include/fs/DeflateStreamSource.h @@ -0,0 +1,41 @@ +#pragma once +#include "fs/IStreamSource.h" +#include "tl/memory_buffer.h" +#include "fs/Api.h" +#include "tl/ptr.h" + +namespace fs +{ + +class FS_API DeflateStreamSource final : public IStreamSource +{ +public: + explicit DeflateStreamSource(IStreamSource& source); //non-owning + explicit DeflateStreamSource(tl::unique_ref source); //owning + ~DeflateStreamSource() override; + + tl::optional getLastError() const override; + size_t read(tl::span data) override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + + tl::unique_ref extract(); + +private: + tl::unique_ptr m_ownedSource; + IStreamSource& m_source; + + mutable tl::optional m_optLastError; + + void initializeDeflate(); + + struct ZLib; + tl::unique_ptr m_zlib; + + tl::memory_buffer m_buffer; +}; + + +} diff --git a/include/fs/Error.h b/include/fs/Error.h new file mode 100644 index 0000000..a344036 --- /dev/null +++ b/include/fs/Error.h @@ -0,0 +1,27 @@ +#pragma once + +#include "tl/result.h" + +namespace fs +{ + +class ISource; +class IStreamSource; +class IMapSource; +class ISink; +class IStreamSink; +class IMapSink; + +enum class ErrorCode : uint8_t +{ + BadPath, + InvalidArgument, + NotFound, + NotAllowed, + SystemError, + NotSupported +}; +typedef tl::error Error; + +} + diff --git a/include/fs/FS.h b/include/fs/FS.h new file mode 100644 index 0000000..c856dd1 --- /dev/null +++ b/include/fs/FS.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include //c78 - fix for error: 'strlen' is not a member of 'std' on android + +#include "tl/platform.h" +#include "tl/string.h" +#include "tl/ptr.h" +#include "tl/result.h" + +//#define FS_H_INCLUDED_CORRECTLY + +//to disable the MSVC warning about inheriting via dominance +#if defined(_MSC_VER) +#pragma warning(disable:4250) +#endif + +#include "fs/Api.h" +#include "Mode.h" +#include "Error.h" + +#include "native/NativeUtils.h" +#include "AbsPath.h" + + +#include "ISource.h" +#include "ISink.h" + +#include "IPack.h" +#include "IWritablePack.h" +#include "IFilesystem.h" + +//native implementation +//#include "native/FolderFilesystem.h" +//#include "native/FolderPack.h" + +#include "native/FileSource.h" +#include "native/FileSink.h" +#include "native/FileMapSource.h" +#include "native/FileMapSink.h" +// +#include "MemorySource.h" +#include "MemorySink.h" + +//clean up the preprocessor mess +#undef FS_H_INCLUDED_CORRECTLY diff --git a/include/fs/FileMapSink.h b/include/fs/FileMapSink.h new file mode 100644 index 0000000..e147f87 --- /dev/null +++ b/include/fs/FileMapSink.h @@ -0,0 +1,42 @@ +#pragma once + +#include "tl/ptr.h" +#include "fs/IMapSink.h" +#include "fs/AbsPath.h" +#include "tl/flag_set2.h" +#include "fs/Mode.h" +#include "fs/Api.h" + +namespace fs +{ + +class NativeFilesystem; + +class FS_API FileMapSink final : public IMapSink +{ + friend class NativeFilesystem; +public: + FileMapSink(const AbsPath& filepath, size_t size, Mode mode = Mode::CreateOrOpenAndClear, Flags flags = Flags()); + ~FileMapSink() override; + + bool isValid() const; + tl::optional getLastError() const override; + + size_t write(tl::span data) override; + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + uint64_t tell() const override; + uint64_t getSize() const override; + tl::span map(size_t size) override; + size_t getPreferredBufferSize() const override; + +private: + uint64_t m_offset = 0; + mutable tl::optional m_optLastError; + + struct Impl; + tl::unique_ptr m_impl; +}; + +} + diff --git a/include/fs/FileMapSource.h b/include/fs/FileMapSource.h new file mode 100644 index 0000000..4aa6911 --- /dev/null +++ b/include/fs/FileMapSource.h @@ -0,0 +1,49 @@ +#pragma once + +#include "tl/ptr.h" + +#include "fs/IMapSource.h" +#include "fs/AbsPath.h" +#include "fs/Api.h" + +namespace fs +{ + +class NativeFilesystem; + +class FS_API FileMapSource final : public IMapSource +{ + friend class NativeFilesystem; +public: + explicit FileMapSource(const AbsPath& filepath); + FileMapSource(const AbsPath& filepath, uint64_t start, size_t size); + + ~FileMapSource() override; + + bool isValid() const; + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + tl::span map(size_t size) override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + +private: + uint64_t m_offset = 0; + mutable tl::optional m_optLastError; + + struct Impl; + tl::unique_ptr m_impl; +}; + +} + diff --git a/include/fs/FileSink.h b/include/fs/FileSink.h new file mode 100644 index 0000000..a55d2f5 --- /dev/null +++ b/include/fs/FileSink.h @@ -0,0 +1,47 @@ +#pragma once + +#include "fs/ISink.h" +#include "fs/Mode.h" +#include "fs/AbsPath.h" +#include "tl/flag_set2.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API FileSink final : public ISink +{ +public: + FileSink(const AbsPath& filepath, Mode mode, Flags flags = Flags()) noexcept; + ~FileSink() noexcept override; + + FileSink(FileSink&& other) noexcept; + FileSink& operator=(FileSink&& other) noexcept; + + bool isValid() const noexcept; + tl::optional getLastError() const override; + + size_t write(tl::span data) override; + + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + size_t getPreferredBufferSize() const override; + + bool resize(uint64_t newSize) noexcept; + +private: + //static tl::result openFile(const AbsPath& filepath, Mode mode, Flags flags) noexcept; + + void close() noexcept; + + bool m_isConstructed = false; + std::array m_arena; + + uint64_t m_offset = 0; + mutable tl::optional m_optLastError; +}; +} diff --git a/include/fs/FileSource.h b/include/fs/FileSource.h new file mode 100644 index 0000000..d7182f4 --- /dev/null +++ b/include/fs/FileSource.h @@ -0,0 +1,49 @@ +#pragma once + +#include "fs/ISource.h" +#include "fs/AbsPath.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API FileSource final : public ISource +{ +public: + explicit FileSource(const AbsPath& i_filepath) noexcept; + FileSource(const AbsPath& i_filepath, Flags i_flags) noexcept; + ~FileSource() noexcept override; + + FileSource(FileSource&& i_other) noexcept; + FileSource& operator=(FileSource&& i_other) noexcept; + + bool isValid() const noexcept; + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t i_offset) override; + void seekRel(int64_t i_offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + +private: + FileSource() noexcept = default; + void swap(FileSource& i_other) noexcept; + + void close() noexcept; + + bool m_isConstructed = false; + std::array m_arena; + + uint64_t m_offset = 0; + mutable uint64_t m_size = uint64_t(-1); + mutable tl::optional m_optLastError; +}; +} + diff --git a/include/fs/FolderPack.h b/include/fs/FolderPack.h new file mode 100644 index 0000000..1236ef1 --- /dev/null +++ b/include/fs/FolderPack.h @@ -0,0 +1,40 @@ +#pragma once + +#include "fs/IPack.h" +#include "fs/Api.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +class FS_API FolderPack : virtual public IPack +{ +public: + explicit FolderPack(AbsPath location); + FolderPack(tl::lent_ref filesystem, AbsPath location); + + ~FolderPack() override = default; + + OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override; + + IsFileResult isFile(const AbsPath& path) const override; + IsFolderResult isFolder(const AbsPath& path) const override; + ExistsResult exists(const AbsPath& path) const override; + + GetStatResult getStat(const AbsPath& path) const override; + + cppcoro::generator enumerate(const AbsPath& path) const override; + cppcoro::generator enumerateRecursively(const AbsPath& path) const override; + + ConvertToNativePathResult convertToNativePath(const AbsPath& path) const override; + +protected: + AbsPath convertToUnderlyingPath(const AbsPath& path) const; + tl::lent_ref m_filesystem; + AbsPath m_location; +}; + +////////////////////////////////////////////////////////////////////////// +} diff --git a/include/fs/IFilesystem.h b/include/fs/IFilesystem.h new file mode 100644 index 0000000..e47b74e --- /dev/null +++ b/include/fs/IFilesystem.h @@ -0,0 +1,126 @@ +#pragma once + +#include "fs/AbsPath.h" +#include "tl/ptr.h" + +#include "fs/Error.h" +#include "fs/Mode.h" + +#include "fs/ISource.h" +#include "fs/IMapSource.h" +#include "fs/ISink.h" +#include "fs/IMapSink.h" +#include "fs/Api.h" +#include + + +namespace fs +{ +using OpenSourceResult = tl::result, Error>; +using OpenStreamSourceResult = tl::result, Error>; +using OpenMapSourceResult = tl::result, Error>; +using OpenSinkResult = tl::result, Error>; +using OpenStreamSinkResult = tl::result, Error>; +using OpenMapSinkResult = tl::result, Error>; +using ConvertToNativePathResult = tl::result; +using IsFileResult = tl::result; +using IsFolderResult = tl::result; +using ExistsResult = tl::result; +using MakeFolderResult = tl::result; +using RenameResult = tl::result; +using RemoveResult = tl::result; +using CopyResult = tl::result; +using MakeHardLinkResult = tl::result; +using MakeSoftLinkResult = tl::result; +using RemoveRecursivelyResult = tl::result; +struct Stat +{ + uint64_t size = 0; + time_t mTime = 0; + time_t aTime = 0; +}; +using GetStatResult = tl::result; +using SetWriteTimeResult = tl::result; + +struct EnumerateEntry +{ + RelPath path; + bool isFolder = false; +}; + +class FS_API IFilesystem +{ +public: + ////////////////////////////////////////////////////////////////////////// + + virtual ~IFilesystem() = default; + + using SourceFlag = ISource::Flag; + using SourceFlags = ISource::Flags; + + struct MapView + { + MapView(uint64_t offset = 0, size_t size = 0) + : offset(offset), size(size) {} + uint64_t offset; + size_t size; //zero size means full size + }; + + //Opens a Read-only Stream + //Searches for the stream in the packs from the mountpoint in top to bottom order, and opens if found + virtual OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const = 0; + virtual OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const = 0; + virtual OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const = 0; + + using SinkFlag = ISink::Flag; + using SinkFlags = ISink::Flags; + + //Opens a Write-only Stream + //Searches for the stream in the packs from the mountpoint in top to bottom order, and opens if found. + //If used with the CREATE flag, it will create the stream if not found. + //NOTE: this will fail if the mountpoint has only read-only packs (zip files, etc) + virtual OpenSinkResult openSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) = 0; + virtual OpenStreamSinkResult openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) = 0; + virtual OpenMapSinkResult openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags = SinkFlags()) = 0; + + ////////////////////////////////////////////////////////////////////////// + + //Returns the physical location of the virtual pack. + virtual ConvertToNativePathResult convertToNativePath(const AbsPath& path) const = 0; + + virtual IsFileResult isFile(const AbsPath& path) const = 0; + virtual IsFolderResult isFolder(const AbsPath& path) const = 0; + virtual ExistsResult exists(const AbsPath& path) const = 0; + + virtual GetStatResult getStat(const AbsPath& path) const = 0; + + virtual MakeFolderResult makeFolder(const AbsPath& path) = 0; + + virtual RenameResult rename(const AbsPath& path, const AbsPath& newPath) = 0; + + //this removes one file or one folder. The folder is removed ONLY if it's empty. If you want to remove a non-empty folder, use RemoveRecursively + virtual RemoveResult remove(const AbsPath& path) = 0; + + virtual CopyResult copy(const AbsPath& path, const AbsPath& newPath) = 0; + + // Hard links are essentially a new name for the same file data. + // The file becomes reference counted and the data will be deleted when all handles to it are removed from the FS. + virtual MakeHardLinkResult makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) = 0; + + // Soft links are like 'weak' pointers to a file. If the original gets deleted the soft link becomes 'null'. + virtual MakeSoftLinkResult makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) = 0; + + //Use this function in order to remove the set folder and all of the files and folders contained inside. + //This function is not atomic; if the removal of any of the folders or files fails, the process will be stopped + //and the function will return false, but any of the previously removed files will be already removed. + virtual RemoveRecursivelyResult removeRecursively(const AbsPath& path) = 0; + + virtual SetWriteTimeResult setWriteTime(const AbsPath& path, time_t time) = 0; + + virtual cppcoro::generator enumerate(const AbsPath& path) const = 0; + virtual cppcoro::generator enumerateRecursively(const AbsPath& path) const = 0; +}; + + +} + diff --git a/include/fs/IMapSink.h b/include/fs/IMapSink.h new file mode 100644 index 0000000..00974d4 --- /dev/null +++ b/include/fs/IMapSink.h @@ -0,0 +1,33 @@ +#pragma once + +#include "ISink.h" +#include "fs/Api.h" + +namespace fs +{ +//Design rationale: +// Sinks are throw-away objects representing a data source or sink (the Sink). +// They are not supposed to be reused. +// There is only one owner of the sink, enforced by the tl::unique_ref wrapper. Due to this, sinks are NOT thread safe. +// There is no open/close for sinks as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects. +// If you need a persistent sink to be able to reload your data, hold a +// path to the sink instead of the sink. This is to allow the underlying file system to change (due to DLC for example). When you need the sink again, request a new one from the file system. + +class FS_API IMapSink : public ISink +{ +public: + IMapSink() = default; + ~IMapSink() override = default; + + IMapSink(IMapSink&&) = default; + IMapSink& operator=(IMapSink&&) = default; + + //this maps the sink from the current pointer and size bytes. the pointer is advanced with size bytes + virtual tl::span map(size_t size) = 0; +}; + + +} + + + diff --git a/include/fs/IMapSource.h b/include/fs/IMapSource.h new file mode 100644 index 0000000..0a83b02 --- /dev/null +++ b/include/fs/IMapSource.h @@ -0,0 +1,32 @@ +#pragma once + +#include "ISource.h" +#include "fs/Api.h" +#include "tl/span.h" + +namespace fs +{ + +//Design rationale: +// Sources are throw-away objects representing a data source or sink (the Sink). +// They are not supposed to be reused. +// There is only one owner of the source, enforced by the tl::unique_ref wrapper. Due to this, sources are NOT thread safe. +// There is no open/close for sources as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects. +// If you need a persistent source to be able to reload your data, hold a +// path to the source instead of the source. This is to allow the underlying file system to change (due to DLC for example). When you need the source again, request a new one from the file system. + +class FS_API IMapSource : public ISource +{ +public: + IMapSource() = default; + ~IMapSource() override = default; + + IMapSource(IMapSource&&) = default; + IMapSource& operator=(IMapSource&&) = default; + + //this maps the source from the current pointer and size bytes. the pointer is advanced with size bytes + virtual tl::span map(size_t size) = 0; +}; + + +} diff --git a/include/fs/IPack.h b/include/fs/IPack.h new file mode 100644 index 0000000..6cb6334 --- /dev/null +++ b/include/fs/IPack.h @@ -0,0 +1,45 @@ +#pragma once + +//#pragma warning( disable : 4250) + +#include "fs/IFilesystem.h" +#include "fs/AbsPath.h" +#include "fs/Error.h" +#include "fs/Mode.h" + +#include "fs/Api.h" + +namespace fs +{ +class ISource; +class IMapSource; + +class FS_API IPack +{ +public: + virtual ~IPack() = default; + + using SourceFlag = IFilesystem::SourceFlag; + using SourceFlags = IFilesystem::SourceFlags; + using MapView = IFilesystem::MapView; + + virtual OpenSourceResult openSource(const AbsPath& i_path, SourceFlags i_flags = SourceFlags()) const = 0; + virtual OpenStreamSourceResult openStreamSource(const AbsPath& i_path, SourceFlags i_flags = SourceFlags()) const = 0; + virtual OpenMapSourceResult openMapSource(const AbsPath& i_path, MapView i_mapView = MapView(), SourceFlags i_flags = SourceFlags()) const = 0; + + virtual IsFileResult isFile(const AbsPath& i_path) const = 0; + virtual IsFolderResult isFolder(const AbsPath& i_path) const = 0; + virtual ExistsResult exists(const AbsPath& i_path) const = 0; + + virtual GetStatResult getStat(const AbsPath& i_path) const = 0; + + virtual cppcoro::generator enumerate(const AbsPath& i_path) const = 0; + virtual cppcoro::generator enumerateRecursively(const AbsPath& i_path) const = 0; + + virtual ConvertToNativePathResult convertToNativePath(const AbsPath& i_path) const = 0; + +protected: + IPack() = default; +}; + +} diff --git a/include/fs/ISink.h b/include/fs/ISink.h new file mode 100644 index 0000000..6b80b5f --- /dev/null +++ b/include/fs/ISink.h @@ -0,0 +1,33 @@ +#pragma once + +#include "fs/IStreamSink.h" +#include "tl/flag_set2.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API ISink : public IStreamSink +{ +public: + enum class Flag + { + }; + typedef tl::flag_set2 Flags; + + ISink() = default; + ~ISink() override = default; + + ISink(ISink&&) = default; + ISink& operator=(ISink&&) = default; + + //changes the sink cursor either to an absolute offset or relative one + //NOTE!!! if the cursor is moved beyond the sink size, the size is increased to fit it. + virtual void seekBeg(uint64_t offset) = 0; + virtual void seekRel(int64_t offset) = 0; + + //returns the sink cursor. Always between 0 and size + virtual uint64_t tell() const = 0; +}; + +} diff --git a/include/fs/ISource.h b/include/fs/ISource.h new file mode 100644 index 0000000..1d5b778 --- /dev/null +++ b/include/fs/ISource.h @@ -0,0 +1,47 @@ +#pragma once + +#include "fs/IStreamSource.h" +#include "fs/Api.h" +#include "tl/flag_set2.h" + +namespace fs +{ + +//Design rationale: +// Sources are throw-away objects representing a data source or sink (the Sink). +// They are not supposed to be reused. +// There is only one owner of the source, enforced by the tl::unique_ref wrapper. Due to this, sources are NOT thread safe. +// There is no open/close for sources as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects. +// If you need a persistent source to be able to reload your data, hold a +// path to the source instead of the source. This is to allow the underlying file system to change (due to DLC for example). When you need the source again, request a new one from the file system. + +class FS_API ISource : public IStreamSource +{ +public: + enum class Flag : uint8_t + { + Unbuffered, + Uncached, + SequentialHint, //takes prio over the random hint flag + RandomHint, + }; + typedef tl::flag_set2 Flags; + + ISource() = default; + ~ISource() override = default; + + ISource(ISource&&) = default; + ISource& operator=(ISource&&) = default; + + //Changes the cursor either to an absolute offset or a relative one + virtual void seekBeg(uint64_t offset) = 0; + virtual void seekRel(int64_t offset) = 0; + + //returns the current cursor position. This is always between 0 and size + virtual uint64_t tell() const = 0; + + //returns the size of the source in bytes + virtual uint64_t getSize() const = 0; +}; + +} diff --git a/include/fs/IStreamSink.h b/include/fs/IStreamSink.h new file mode 100644 index 0000000..6d922a8 --- /dev/null +++ b/include/fs/IStreamSink.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include "fs/Error.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API IStreamSink +{ +public: + IStreamSink() = default; + virtual ~IStreamSink() = default; + + IStreamSink(IStreamSink&&) = default; + IStreamSink& operator=(IStreamSink&&) = default; + + //This will return the last generated error AND CLEAR IT AFTERWARDS. + virtual tl::optional getLastError() const = 0; + + //writes size bytes from the buffer to the sink at the current cursor position. + //the sink size is increased if the sinks reaches the end + virtual size_t write(tl::span data) = 0; + + //returns the size of the sink. Always >= tell + virtual uint64_t getSize() const = 0; + + // The preferred buffer size for writes. Use this buffer size for optimal performance + // If zero, use whatever you want + virtual size_t getPreferredBufferSize() const = 0; +}; + +template +struct StreamSinkSerializationHelperWrite_Internal; + +template +struct StreamSinkSerializationHelperWrite_Internal +{ + static IStreamSink& write(IStreamSink& s, const T& val) + { + static_assert(std::is_standard_layout_v && !std::is_pointer_v && !std::is_array_v, "You need a specialized write for T"); + s.write({ reinterpret_cast(&val), sizeof(T) }); + return s; + } + + static IStreamSink& write(IStreamSink& s, tl::span val) + { + const uint32_t count = static_cast(val.size()); + s.write({ reinterpret_cast(&count), sizeof(count) }); + for(const T& v : val) + s.write( { reinterpret_cast(&v), sizeof(T) }); + return s; + } +}; + +// specialization to read enums +template +struct StreamSinkSerializationHelperWrite_Internal +{ + static IStreamSink& write(IStreamSink& s, const EnumType& val) + { + // first read into the temporary of the underlying type + using underlying_t = std::underlying_type_t; + underlying_t underlyingValue = static_cast(val); + + IStreamSink& ret = StreamSinkSerializationHelperWrite_Internal::write(s, underlyingValue); + return ret; + } +}; + +template +struct SimpleStreamSerializationHelperWrite : StreamSinkSerializationHelperWrite_Internal>> +{ +}; + +template +IStreamSink& operator<<(IStreamSink& s, const T& val) +{ + return SimpleStreamSerializationHelperWrite::write(s, val); +} + +template +IStreamSink& operator<<(IStreamSink& s, tl::span val) +{ + return SimpleStreamSerializationHelperWrite::write(s, val); +} + +} + + diff --git a/include/fs/IStreamSource.h b/include/fs/IStreamSource.h new file mode 100644 index 0000000..16b49c3 --- /dev/null +++ b/include/fs/IStreamSource.h @@ -0,0 +1,101 @@ +#pragma once + +#include "tl/memory_buffer.h" +#include +#include +#include "tl/result.h" +#include "fs/Error.h" +#include "fs/Api.h" + + +namespace fs +{ + +//Design rationale: +// Throw-away objects representing a non-seekable, non-sized data source or sink. +// They are not supposed to be reused. +// There is only one owner of the source, enforced by the tl::unique_ref wrapper. Due to this, simple sources are NOT thread safe. +// There is no open/close for simple sources as the open is done in the constructor and close in the destructor. This is to reinforce the idea of throw-away objects. +// If you need a persistent source to be able to reload your data, hold a +// path to the source instead of the source. This is to allow the underlying file system to change (due to DLC for example). When you need the source again, request a new one from the file system. + +class FS_API IStreamSource +{ +public: + IStreamSource() = default; + virtual ~IStreamSource() = default; + + IStreamSource(IStreamSource&&) = default; + IStreamSource& operator=(IStreamSource&&) = default; + + //This will return the last generated error AND CLEAR IT AFTERWARDS. + virtual tl::optional getLastError() const = 0; + + //Reads from the source into the buffer size bytes. The source pointer is advanced + //Returns the number of bytes successfully read + virtual size_t read(tl::span data) = 0; + + virtual bool isEOS() const = 0; + + // The preferred buffer size for reads. Use this buffer size for optimal performance + // If zero, use whatever you want + virtual size_t getPreferredBufferSize() const = 0; +}; + +template +struct StreamSourceSerializationHelper_Internal; + +template +struct StreamSourceSerializationHelper_Internal +{ + static IStreamSource& read(IStreamSource& s, T& val) + { + static_assert(std::is_fundamental_v && !std::is_pointer_v && !std::is_array_v, "You need a specialized read for T"); + s.read({ reinterpret_cast(&val), sizeof(T) }); + return s; + } + + static IStreamSource& read(IStreamSource& s, tl::memory_buffer& val) + { + uint32_t count = 0; + s.read({ reinterpret_cast(&count), sizeof(count) }); + val.resize(count); + s.read({ val.data(), count }); + return s; + } +}; + +// specialization to read enums +template +struct StreamSourceSerializationHelper_Internal +{ + static IStreamSource& read(IStreamSource& s, EnumType& val) + { + // first read into the temporary of the underlying type + using underlying_t = std::underlying_type_t; + underlying_t underlyingValue; + + IStreamSource& ret = StreamSourceSerializationHelper_Internal::read(s, underlyingValue); + // put the read value into the enum + val = EnumType(underlyingValue); + return ret; + } +}; + +template +struct SimpleStreamSerializationHelper : StreamSourceSerializationHelper_Internal>> +{ +}; + +template +IStreamSource& operator>>(IStreamSource& s, T& val) +{ + return SimpleStreamSerializationHelper::read(s, val); +} + +inline IStreamSource& operator>>(IStreamSource& s, tl::memory_buffer& val) +{ + return SimpleStreamSerializationHelper::read(s, val); +} + +} diff --git a/include/fs/IWritablePack.h b/include/fs/IWritablePack.h new file mode 100644 index 0000000..d1d6a1f --- /dev/null +++ b/include/fs/IWritablePack.h @@ -0,0 +1,29 @@ +#pragma once + +#include "IPack.h" +#include "Mode.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API IWritablePack : virtual public IPack +{ +public: + using SinkFlag = IFilesystem::SinkFlag; + using SinkFlags = IFilesystem::SinkFlags; + + virtual OpenSinkResult openSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) = 0; + virtual OpenStreamSinkResult openStreamSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) = 0; + virtual OpenMapSinkResult openMapSink(const AbsPath& i_path, Mode i_mode, size_t i_size, SinkFlags i_flags = SinkFlags()) = 0; + + virtual RemoveResult remove(const AbsPath& i_path) = 0; + virtual RenameResult rename(const AbsPath& i_path, const AbsPath& i_newPath) = 0; + + virtual MakeFolderResult makeFolder(const AbsPath& i_path) = 0; + virtual RemoveRecursivelyResult removeRecursively(const AbsPath& i_path) = 0; + + virtual SetWriteTimeResult setWriteTime(const AbsPath& i_path, time_t i_time) = 0; +}; + +} diff --git a/include/fs/MapSourceView.h b/include/fs/MapSourceView.h new file mode 100644 index 0000000..e242c55 --- /dev/null +++ b/include/fs/MapSourceView.h @@ -0,0 +1,42 @@ +#pragma once + +#include "tl/ptr.h" +#include "IMapSource.h" +#include "MemoryViewSource.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API MapSourceView final : public IMapSource +{ +public: + + MapSourceView(tl::lent_ref srcSource, uint64_t offset, size_t size); + + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + tl::span map(size_t size) override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + +private: + + tl::span m_memView; + MemoryViewSource m_memViewSource; + + tl::span constructMemView(IMapSource& srcSource, uint64_t offset, size_t size) const; +}; + +} + diff --git a/include/fs/MemorySink.h b/include/fs/MemorySink.h new file mode 100644 index 0000000..64d1084 --- /dev/null +++ b/include/fs/MemorySink.h @@ -0,0 +1,56 @@ +#pragma once + +#include "fs/IMapSink.h" +#include "fs/Error.h" +#include "tl/memory_buffer.h" +#include "fs/Api.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +class MemorySource; + +class FS_API MemorySink final : public IMapSink +{ +public: + MemorySink(size_t reserve = BUFSIZ); + explicit MemorySink(tl::memory_buffer&& buffer); + ~MemorySink() override = default; + + MemorySink(MemorySink&& sink) = default; + MemorySink& operator=(MemorySink&& sink) = default; + + tl::optional getLastError() const override; + + size_t write(tl::span data) override; + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + uint64_t tell() const override; + uint64_t getSize() const override; + tl::span getDataView() const; + tl::memory_buffer getDataCopy() const; + tl::memory_buffer getDataMove(); + void clear(); + void swap(MemorySource& source); + void reserve(size_t capacity); + void shrinkToFit(); + + tl::span map(size_t size) override; + + size_t getPreferredBufferSize() const override; + + //this applies OVER the current size + void preReserve(size_t size); + + static tl::memory_buffer extractData(MemorySink sink); + +private: + size_t m_offset = 0; + tl::memory_buffer m_data; + mutable tl::optional m_optLastError; +}; + +////////////////////////////////////////////////////////////////////////// +} + diff --git a/include/fs/MemorySource.h b/include/fs/MemorySource.h new file mode 100644 index 0000000..5e56bf8 --- /dev/null +++ b/include/fs/MemorySource.h @@ -0,0 +1,55 @@ +#pragma once + +#include "IMapSource.h" +#include "tl/memory_buffer.h" +#include "fs/Api.h" + +namespace fs +{ +class MemorySink; + +class FS_API MemorySource final : public IMapSource +{ + friend class MemorySink; +public: + // The source will make an internal copy of the data passed by the constructor + // rawMemory can be safely destroyed after calling the constructor + explicit MemorySource(tl::span data); + explicit MemorySource(MemorySink& sink); + explicit MemorySource(const tl::memory_buffer& memoryBuffer); + + MemorySource(MemorySource&&) = default; + MemorySource& operator=(MemorySource&&) = default; + + //we have another constructor for empty source + MemorySource() = default; + ~MemorySource() override = default; + + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + tl::span map(size_t size) override; + + void swap(MemorySink& sink); + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + +protected: + tl::memory_buffer m_data; + +private: + size_t m_offset = 0; + mutable tl::optional m_optLastError; +}; + +} + diff --git a/include/fs/MemoryViewSource.h b/include/fs/MemoryViewSource.h new file mode 100644 index 0000000..760fd7a --- /dev/null +++ b/include/fs/MemoryViewSource.h @@ -0,0 +1,47 @@ +#pragma once + +#include "IMapSource.h" +#include "fs/Api.h" +#include "tl/optional.h" + +namespace fs +{ + +class FS_API MemoryViewSource final : public IMapSource +{ +public: + ~MemoryViewSource() override = default; + + explicit MemoryViewSource(tl::span memView); + + MemoryViewSource(MemoryViewSource&&) = default; + MemoryViewSource& operator=(MemoryViewSource&&) = default; + + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t offset) override; + void seekRel(int64_t offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + tl::span map(size_t size) override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + +protected: + + tl::span m_memView; + +private: + uint64_t m_offset = 0; + mutable tl::optional m_optLastError; + +}; + +} + diff --git a/include/fs/Mode.h b/include/fs/Mode.h new file mode 100644 index 0000000..e8a1515 --- /dev/null +++ b/include/fs/Mode.h @@ -0,0 +1,16 @@ +#pragma once + +namespace fs +{ + +enum class Mode +{ + CreateOrOpen, //Create if doesn't exist, or open otherwise, but DON'T clear the file + CreateOrOpenAndClear, //Create if doesn't exist, or open otherwise, and clear the file + CreateIfNew, //Create the file ONLY if it doesn't exist already. Otherwise FAIL + + Open, //Open if exists, fail otherwise, but DON'T clear the file + OpenAndClear //Open the file if exists, fail otherwise, and clear the file +}; + +} diff --git a/include/fs/NativeFilesystem.h b/include/fs/NativeFilesystem.h new file mode 100644 index 0000000..f7ebde9 --- /dev/null +++ b/include/fs/NativeFilesystem.h @@ -0,0 +1,82 @@ +#pragma once + +#include "IFilesystem.h" +#include "fs/Error.h" +#include "fs/Mode.h" +#include "fs/Api.h" +#include "tl/vector.h" + +namespace fs +{ +class FS_API NativeFilesystem final : public IFilesystem, public tl::lendable_base +{ +public: + + enum class CheckFlag : uint8_t + { + ValidatePathCase, //Win32 only, relatively slow, but makes the filesystem behave more like on posix (case-sensitive filenames) + }; + using CheckFlags = tl::flag_set2; + + NativeFilesystem() = default; + explicit NativeFilesystem(CheckFlags checkFlags); + + OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override; + OpenSinkResult openSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override; + OpenStreamSinkResult openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags = SinkFlags()) override; + OpenMapSinkResult openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags = SinkFlags()) override; + + ConvertToNativePathResult convertToNativePath(const AbsPath& path) const override; + + IsFileResult isFile(const AbsPath& path) const override; + IsFolderResult isFolder(const AbsPath& path) const override; + ExistsResult exists(const AbsPath& path) const override; + + MakeFolderResult makeFolder(const AbsPath& path) override; + + RenameResult rename(const AbsPath& path, const AbsPath& newPath) override; + + RemoveResult remove(const AbsPath& path) override; + RemoveRecursivelyResult removeRecursively(const AbsPath& path) override; + + CopyResult copy(const AbsPath& path, const AbsPath& newPath) override; + + MakeHardLinkResult makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) override; + MakeSoftLinkResult makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) override; + + SetWriteTimeResult setWriteTime(const AbsPath& path, time_t time) override; + + GetStatResult getStat(const AbsPath& path) const override; + + cppcoro::generator enumerate(const AbsPath& path) const override; + cppcoro::generator enumerateRecursively(const AbsPath& path) const override; + + AbsPath getCurrentFolder() const; + tl::result setCurrentFolder(const AbsPath& path); + + typedef tl::result, Error> EnumerateFilesResult; + EnumerateFilesResult enumerateFiles(const AbsPath& fullPath) const; + + typedef tl::result, Error> EnumerateFoldersResult; + EnumerateFoldersResult enumerateFolders(const AbsPath& fullPath) const; + + AbsPath makeAbsPath(const tl::string& string) const; + + +private: + CheckFlags m_checkFlags; +}; + +//this FS will do various checks to make the platforms behave similarly +//For example path case checks on Win32 to make its FS case-sensitive +FS_API extern NativeFilesystem native; + +//this FS is RAW with no advanced (read: slow) checks +//Use it where appropriate, where native is clearly slowing down something vital. +//Normal use-case: tools +//*** For game or library use, consult with the lead, and think 3 times. *** +FS_API extern NativeFilesystem nativeRaw; + +} diff --git a/include/fs/NativeUtils.h b/include/fs/NativeUtils.h new file mode 100644 index 0000000..78503f9 --- /dev/null +++ b/include/fs/NativeUtils.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include "AbsPath.h" + +namespace fs +{ +namespace native_utils +{ + +bool validateCaseSensitiveFilename(const AbsPath& filePath) noexcept; +bool validateCaseSensitiveFolder(const AbsPath& folderPath) noexcept; + +typedef eastl::fixed_string wstring_t; +typedef eastl::fixed_string string_t; + +wstring_t utf8To16(tl::span str) noexcept; +string_t utf16To8(tl::span str) noexcept; + +//////////////////////////////////////////////////////////////////////// + +} +} diff --git a/include/fs/ProcessStream.h b/include/fs/ProcessStream.h new file mode 100644 index 0000000..54971e2 --- /dev/null +++ b/include/fs/ProcessStream.h @@ -0,0 +1,14 @@ +#pragma once +#include "Error.h" +#include "fs/Api.h" +#include "tl/result.h" + +namespace fs +{ +class IStreamSource; + +typedef tl::result ProcessStreamResult; +typedef tl::function data)> ProcessDataCallback; +// This API Reads the provided stream and pipes it through the provided data callback to allow implementation of custom filters/encodings. +FS_API ProcessStreamResult processStream(IStreamSource& source, const ProcessDataCallback& dataCallback, size_t bufferSize = 0); +} diff --git a/include/fs/ReadStream.h b/include/fs/ReadStream.h new file mode 100644 index 0000000..4cc7724 --- /dev/null +++ b/include/fs/ReadStream.h @@ -0,0 +1,20 @@ +#pragma once +#include "Error.h" +#include "tl/memory_buffer.h" +#include "fs/Api.h" +#include "IFilesystem.h" + +namespace fs +{ +class IStreamSource; + +typedef tl::result ReadStreamResult; +FS_API ReadStreamResult readStream(tl::memory_buffer& buffer, IStreamSource& source, size_t bufferSize = 0); +FS_API ReadStreamResult readStream(std::string& buffer, IStreamSource& source, size_t bufferSize = 0); +FS_API ReadStreamResult readStream(eastl::string& buffer, IStreamSource& source, size_t bufferSize = 0); + +FS_API ReadStreamResult readFile(tl::memory_buffer& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0); +FS_API ReadStreamResult readFile(std::string& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0); +FS_API ReadStreamResult readFile(eastl::string& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize = 0); + +} diff --git a/include/fs/SourceView.h b/include/fs/SourceView.h new file mode 100644 index 0000000..7bd973f --- /dev/null +++ b/include/fs/SourceView.h @@ -0,0 +1,39 @@ +#pragma once + +#include "ISource.h" +#include +#include "fs/Api.h" + +namespace fs +{ + +class FS_API SourceView final : public ISource +{ +public: + SourceView(tl::unique_ref&& i_srcSource, uint64_t i_offset, uint64_t i_size); + ~SourceView() override = default; + + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t i_offset) override; + void seekRel(int64_t i_offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + +private: + tl::unique_ref m_srcSource; + mutable tl::optional m_optLastError; + uint64_t m_viewOffset = 0; + uint64_t m_viewSize = 0; + uint64_t m_offset = 0; +}; + +} + diff --git a/include/fs/StreamSourceToSourceAdapter.h b/include/fs/StreamSourceToSourceAdapter.h new file mode 100644 index 0000000..3ce3a54 --- /dev/null +++ b/include/fs/StreamSourceToSourceAdapter.h @@ -0,0 +1,43 @@ +#pragma once +#include "IMapSource.h" +#include "IStreamSource.h" +#include "tl/memory_buffer.h" +#include "fs/Api.h" +#include "tl/ptr.h" + +namespace fs +{ + +class FS_API StreamSourceToSourceAdapter final : public IMapSource +{ +public: + explicit StreamSourceToSourceAdapter(tl::unique_ref&& i_srcSource); + ~StreamSourceToSourceAdapter() override = default; + + tl::optional getLastError() const override; + + size_t read(tl::span data) override; + + void seekBeg(uint64_t i_offset) override; + void seekRel(int64_t i_offset) override; + + uint64_t tell() const override; + uint64_t getSize() const override; + + bool isEOS() const override; + + size_t getPreferredBufferSize() const override; + + tl::span map(size_t size) override; + +private: + void readAll() const; + + tl::unique_ref m_srcSource; + mutable tl::optional m_optLastError; + mutable tl::memory_buffer m_data; + mutable bool m_readAll = false; + uint64_t m_offset = 0; +}; + +} diff --git a/include/fs/WritableFolderPack.h b/include/fs/WritableFolderPack.h new file mode 100644 index 0000000..92aecb3 --- /dev/null +++ b/include/fs/WritableFolderPack.h @@ -0,0 +1,32 @@ +#pragma once + +#include "fs/IWritablePack.h" +#include "fs/FolderPack.h" +#include "fs/Error.h" +#include "fs/Mode.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API WritableFolderPack final : virtual public IWritablePack, public FolderPack +{ +public: + explicit WritableFolderPack(AbsPath location); + WritableFolderPack(tl::lent_ref filesystem, AbsPath location); + ~WritableFolderPack() override = default; + + OpenSinkResult openSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) override; + OpenStreamSinkResult openStreamSink(const AbsPath& i_path, Mode i_mode, SinkFlags i_flags = SinkFlags()) override; + OpenMapSinkResult openMapSink(const AbsPath& i_path, Mode i_mode, size_t i_size, SinkFlags i_flags = SinkFlags()) override; + + RemoveResult remove(const AbsPath& i_path) override; + RenameResult rename(const AbsPath& i_path, const AbsPath& i_newPath) override; + + MakeFolderResult makeFolder(const AbsPath& i_path) override; + RemoveRecursivelyResult removeRecursively(const AbsPath& i_path) override; + + SetWriteTimeResult setWriteTime(const AbsPath& i_path, time_t i_time) override; +}; + +} diff --git a/include/fs/zip/DeflateFileWriter.h b/include/fs/zip/DeflateFileWriter.h new file mode 100644 index 0000000..bce5ffd --- /dev/null +++ b/include/fs/zip/DeflateFileWriter.h @@ -0,0 +1,37 @@ +#pragma once + +#include "fs/AbsPath.h" +#include "fs/zip/ZipWriter.h" + +#include "tl/ptr.h" + +namespace fs +{ + +class IFilesystem; + +////////////////////////////////////////////////////////////////////////// + +class FS_API DeflateFileWriter +{ +public: + //compression level: 0 to 9 + DeflateFileWriter(AbsPath i_filePath, tl::lent_ref i_filesystem, uint8_t i_compressionLevel); + + DeflateFileWriter(const DeflateFileWriter&) = delete; + DeflateFileWriter& operator=(const DeflateFileWriter&) = delete; + + DeflateFileWriter(DeflateFileWriter&&) = default; + DeflateFileWriter& operator=(DeflateFileWriter&&) = default; + + ZipWriter::DataWriterResult operator()(IStreamSink& i_sink); + +private: + AbsPath m_filePath; + tl::lent_ref m_filesystem; + uint8_t m_compressionLevel = 9; +}; + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/include/fs/zip/DeflateSinkWriter.h b/include/fs/zip/DeflateSinkWriter.h new file mode 100644 index 0000000..2812e22 --- /dev/null +++ b/include/fs/zip/DeflateSinkWriter.h @@ -0,0 +1,37 @@ +#pragma once + +#include "fs/IStreamSource.h" +#include "fs/zip/ZipWriter.h" + +namespace fs +{ + +////////////////////////////////////////////////////////////////////////// + +class ISource; +////////////////////////////////////////////////////////////////////////// + +class FS_API DeflateSinkWriter +{ +public: + //compression level: 0 to 9 + DeflateSinkWriter(IStreamSource& source, uint8_t compressionLevel); + DeflateSinkWriter(tl::span data, uint8_t compressionLevel); + + DeflateSinkWriter(const DeflateSinkWriter&) = delete; + DeflateSinkWriter& operator=(const DeflateSinkWriter&) = delete; + + DeflateSinkWriter(DeflateSinkWriter&&) = default; + DeflateSinkWriter& operator=(DeflateSinkWriter&&) = default; + + ZipWriter::DataWriterResult operator()(IStreamSink& sink); + +private: + IStreamSource* m_source = nullptr; + tl::span m_data; + uint8_t m_compressionLevel = 9; +}; + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/include/fs/zip/StoreFileWriter.h b/include/fs/zip/StoreFileWriter.h new file mode 100644 index 0000000..17cd15d --- /dev/null +++ b/include/fs/zip/StoreFileWriter.h @@ -0,0 +1,35 @@ +#pragma once + +#include "fs/AbsPath.h" +#include "fs/zip/ZipWriter.h" + +#include "tl/ptr.h" + +namespace fs +{ + +class IFilesystem; + +////////////////////////////////////////////////////////////////////////// + +class FS_API StoreFileWriter +{ +public: + StoreFileWriter(AbsPath i_filePath, tl::lent_ref i_filesystem); + + StoreFileWriter(const StoreFileWriter&) = delete; + StoreFileWriter& operator=(const StoreFileWriter&) = delete; + + StoreFileWriter(StoreFileWriter&&) = default; + StoreFileWriter& operator=(StoreFileWriter&&) = default; + + ZipWriter::DataWriterResult operator()(IStreamSink& i_sink); + +private: + AbsPath m_filePath; + tl::lent_ref m_filesystem; +}; + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/include/fs/zip/StoreSinkWriter.h b/include/fs/zip/StoreSinkWriter.h new file mode 100644 index 0000000..f4c5c5f --- /dev/null +++ b/include/fs/zip/StoreSinkWriter.h @@ -0,0 +1,32 @@ +#pragma once + +#include "fs/zip/ZipWriter.h" + +namespace fs +{ + +////////////////////////////////////////////////////////////////////////// + +class ISource; +////////////////////////////////////////////////////////////////////////// + +class FS_API StoreSinkWriter +{ +public: + explicit StoreSinkWriter(ISource& source); + + StoreSinkWriter(const StoreSinkWriter&) = delete; + StoreSinkWriter& operator=(const StoreSinkWriter&) = delete; + + StoreSinkWriter(StoreSinkWriter&&) = default; + StoreSinkWriter& operator=(StoreSinkWriter&&) = default; + + ZipWriter::DataWriterResult operator()(IStreamSink& sink); + +private: + tl::reference_wrapper m_inSource; +}; + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/include/fs/zip/ZipBase.h b/include/fs/zip/ZipBase.h new file mode 100644 index 0000000..16a23e8 --- /dev/null +++ b/include/fs/zip/ZipBase.h @@ -0,0 +1,158 @@ +#pragma once +#include "tl/flag_set.h" +#include "fs/Api.h" + +namespace fs +{ + +class FS_API ZipBase +{ +public: + enum class GeneralBitFlag : uint16_t + { + Encrypted = 1 << 0, + CompressionMethodSpecific1 = 1 << 1, + CompressionMethodSpecific2 = 1 << 2, + DataDescriptorNeeded = 1 << 3, + StrongEncryption = 1 << 6, + LanguageEncodingFlag = 1 << 11, + LocalHeaderFieldsMasked = 1 << 13, + }; + typedef tl::flag_set GeneralBitFlags; + + enum class CompressionMethod : uint16_t + { + Store = 0, + Shrunk = 1, + Reduced1 = 2, + Reduced2 = 3, + Reduced3 = 4, + Reduced4 = 5, + Implode = 6, + //Reserved = 7, + Deflate = 8, + Deflate64 = 9, + PKWareImplode = 10, + //Reserved = 11, + BZip2 = 12, + //Reserved = 13, + LZMA = 14, + //Reserved = 15, + //Reserved = 16, + //Reserved = 17, + Terse = 18, + LZ77 = 19, + + LZ4 = 20, // non-standard + ZStd = 21, // non-standard + + WavPack = 97, + PPMd = 98, + }; + +protected: + ////////////////////////////////////////////////////////////////////////// + //These structs follow pretty closely the https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT standard + +#pragma pack(push, 1) + + struct LocalFileHeader + { + static constexpr uint32_t k_signature = 0x04034b50u; + uint32_t signature = k_signature; + uint16_t versionNeededToExtract = 0; + uint16_t generalBitFlag = 0; + uint16_t compressionMethod = 0; + uint16_t lastModFileTime = 0; + uint16_t lastModFileDate = 0; + uint32_t crc32 = 0; + uint32_t compressedSize = 0; + uint32_t uncompressedSize = 0; + uint16_t filenameLength = 0; + uint16_t extraFieldLength = 0; + }; + + struct DataDescriptor + { + uint32_t crc32 = 0; + uint32_t compressedSize = 0; + uint32_t uncompressedSize = 0; + }; + + struct CentralDirectoryFileHeader + { + static constexpr uint32_t k_signature = 0x02014b50u; + uint32_t signature = k_signature; + uint16_t versionMadeBy = 0; + uint16_t versionNeededToExtract = 0; + uint16_t generalBitFlag = 0; + uint16_t compressionMethod = 0; + uint16_t lastModFileTime = 0; + uint16_t lastModFileDate = 0; + uint32_t crc32 = 0; + uint32_t compressedSize = 0; + uint32_t uncompressedSize = 0; + uint16_t filenameLength = 0; + uint16_t extraFieldLength = 0; + uint16_t fileCommentLength = 0; + uint16_t diskNumberStart = 0; + uint16_t internalFileAttributes = 0; + uint32_t externalFileAttributes = 0; + uint32_t localHeaderOffset = 0; + }; + + struct EndOfCentralDirectoryRecord + { + static constexpr uint32_t k_signature = 0x06054b50u; + uint32_t signature = k_signature; + uint16_t diskNumber = 0; + uint16_t centralDirectoryStartDiskNumber = 0; + uint16_t centralDirectoryRecordCountOnThisDisk = 0; + uint16_t centralDirectoryRecordCount = 0; + uint32_t centralDirectorySize = 0; + uint32_t centralDirectoryOffset = 0; + uint16_t zipFileCommentLength = 0; + }; + + struct ExtendedInformationExtraField64 + { + static constexpr uint32_t k_tag = 0x0001; + uint16_t tag = k_tag; + uint16_t size = 0; + uint64_t originalSize = 0; + uint64_t compressedSize = 0; + uint64_t localHeaderOffset = 0; + uint32_t diskNumber = 0; + }; + + struct EndOfCentralDirectoryRecord64 + { + static constexpr uint32_t k_signature = 0x06064b50u; + uint32_t signature = k_signature; + uint64_t endOfCentralDirectorySize = 0; + uint16_t versionMadeBy = 0; + uint16_t versionNeededToExtract = 0; + uint32_t diskNumber = 0; + uint32_t centralDirectoryStartDiskNumber = 0; + uint64_t centralDirectoryRecordCountOnThisDisk = 0; + uint64_t centralDirectoryRecordCount = 0; + uint64_t centralDirectorySize = 0; + uint64_t centralDirectoryOffset = 0; + }; + + struct EndOfCentralDirectoryLocator64 + { + static constexpr uint32_t k_signature = 0x07064b50u; + uint32_t signature = k_signature; + uint32_t centralDirectoryStartDiskNumber = 0; + uint64_t endOfCentralDirectoryRecordOffset = 0; + uint32_t diskCount = 0; + }; + +#pragma pack(pop) + + static void computeMsDosTime(time_t time, uint16_t& o_date, uint16_t& o_time); + static CentralDirectoryFileHeader computeCentralDirectoryFileHeader(const LocalFileHeader& header, bool zip64); +}; + +} diff --git a/include/fs/zip/ZipPack.h b/include/fs/zip/ZipPack.h new file mode 100644 index 0000000..1b24d53 --- /dev/null +++ b/include/fs/zip/ZipPack.h @@ -0,0 +1,121 @@ +#pragma once + +#include "tl/result.h" +#include "tl/string.h" +#include "tl/vector_map.h" + +#include "fs/IPack.h" +#include "fs/AbsPath.h" +#include "fs/zip/ZipReader.h" + +#include "fs/Api.h" + +namespace fs +{ +class MapSourceView; + +class FS_API ZipPack final : virtual public IPack +{ +public: + ZipPack() = default; + ~ZipPack() override = default; + + typedef tl::result, Error> CreateResult; + + static CreateResult create(tl::unique_ref zipFileSource); + static CreateResult create(tl::lent_ref filesystem, AbsPath zipFileLocation); + + virtual void setEncryptionData(const tl::string& key, uint32_t rounds); + + OpenSourceResult openSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenStreamSourceResult openStreamSource(const AbsPath& path, SourceFlags flags = SourceFlags()) const override; + OpenMapSourceResult openMapSource(const AbsPath& path, MapView mapView = MapView(), SourceFlags flags = SourceFlags()) const override; + + IsFileResult isFile(const AbsPath& path) const override; + IsFolderResult isFolder(const AbsPath& path) const override; + ExistsResult exists(const AbsPath& path) const override; + + GetStatResult getStat(const AbsPath& path) const override; + + cppcoro::generator enumerate(const AbsPath& path) const override; + cppcoro::generator enumerateRecursively(const AbsPath& path) const override; + + tl::result convertToNativePath(const AbsPath& path) const override; + +protected: + struct File; + struct Folder; + struct EntryIndex; + + ZipPack(tl::unique_ref zipFileSource, ZipReader zipReader); + ZipPack(tl::lent_ref filesystem, AbsPath zipFileLocation, ZipReader zipReader); + + OpenMapSourceResult openRawMapSource(const File& file, MapView mapView) const; + OpenSourceResult openRawSource(const File& file) const; + + OpenSourceResult openZipSource(const File& file) const; + OpenMapSourceResult openZipMapSource(const File& file) const; + OpenStreamSourceResult openZipStreamSource(const File& file) const; + + uint16_t getOrAddFolder(AbsPath folderPath, tl::vector>& io_childrenIndices); + void createEntries(ZipReader zipReader); + + struct File + { + tl::string name; + uint8_t isCompressed : 1; + uint8_t uncompressedSizeH = 0; + uint8_t compressedSizeH = 0; + uint8_t dataOffsetH = 0; + uint16_t parentIndex = 0; + uint32_t lastModTimePoint = 0; + uint32_t uncompressedSizeL = 0; + uint32_t compressedSizeL = 0; + uint32_t dataOffsetL = 0; + }; + static_assert(sizeof(File) <= sizeof(tl::string) + 40, "Check your sizes!!!"); + + struct Folder + { + tl::string name; + uint16_t parentIndex = 0; + uint16_t childrenStartIndex = 0; // this is an index inside m_childrenIndices + uint16_t childrenCount = 0; + uint32_t lastModTimePoint = 0; + }; + static_assert(sizeof(Folder) <= sizeof(tl::string) + 20, "Check your sizes!!!"); + + struct EntryIndex + { + EntryIndex() = default; + EntryIndex(bool isFolder, uint16_t index) + : isFolder(isFolder) + , index(index) + { + } + uint32_t isFolder : 1; + uint32_t index : 16; + }; + static_assert(sizeof(EntryIndex) == 4, "Check your sizes!!!"); + +private: + tl::lent_ptr m_filesystem; + + mutable std::mutex m_zipMapSourceLock; + tl::unique_ptr m_zipMapSource; + + AbsPath m_zipFileLocation; + + tl::string m_encryptionKey; + uint32_t m_encryptionRounds = 32; + + tl::vector_map m_pathToIndex; + tl::vector m_files; + tl::vector m_folders; + tl::vector m_childrenIndices; // these are indices inside the m_files or m_folders + + void buildUpPath(RelPath& path, const Folder& folder, uint32_t toIndex) const; + void buildUpPath(RelPath& path, const File& file, uint32_t toIndex) const; +}; + +} diff --git a/include/fs/zip/ZipReader.h b/include/fs/zip/ZipReader.h new file mode 100644 index 0000000..2e525a9 --- /dev/null +++ b/include/fs/zip/ZipReader.h @@ -0,0 +1,63 @@ +#pragma once +#include "ZipBase.h" +#include "tl/memory_buffer.h" +#include "tl/result.h" +#include "tl/vector.h" + +namespace fs +{ + +////////////////////////////////////////////////////////////////////////// + +class ISource; + +////////////////////////////////////////////////////////////////////////// + +class ZipReader : public ZipBase +{ +public: + enum class ErrorCode : uint8_t + { + InvalidStream, + BadSignature, + BadOffset, + CorruptedFile, + }; + typedef tl::error Error; + + typedef tl::result CreateResult; + + static CreateResult create(ISource& source); + + struct Entry + { + tl::string name; + CompressionMethod compressionMethod = CompressionMethod::Store; + GeneralBitFlags generalBitFlags; + uint64_t uncompressedSize = 0; + uint64_t compressedSize = 0; + uint32_t crc32 = 0; + uint64_t localHeaderOffset = 0; + uint64_t dataOffset = 0; + time_t lastModTimePoint = 0; + tl::memory_buffer comment; + }; + + size_t getEntryCount() const; + const Entry& getEntry(size_t index) const; + + typedef tl::result SeekResult; + SeekResult seekSourceToEntryData(const Entry& entry, ISource& o_source) const; + +private: + explicit ZipReader(tl::vector entries); + + typedef tl::result, Error> CentralDirectoryResult; + static CentralDirectoryResult findCentralDirectoryBounds(ISource& source); + + tl::vector m_entries; +}; + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/include/fs/zip/ZipUtils.h b/include/fs/zip/ZipUtils.h new file mode 100644 index 0000000..eacda52 --- /dev/null +++ b/include/fs/zip/ZipUtils.h @@ -0,0 +1,19 @@ +#pragma once +#include "fs/Error.h" +#include "fs/Api.h" +#include "fs/IFilesystem.h" + +namespace fs +{ + +class ISink; + +typedef tl::result UnzipResult; +FS_API UnzipResult unzipSource(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::unique_ref source); +FS_API UnzipResult unzipFile(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::lent_ref filesystem, AbsPath srcFilePath); + +typedef tl::result ZipResult; +FS_API ZipResult zipToSink(ISink& sink, tl::lent_ref filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment = 0); +FS_API ZipResult zipToFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, tl::lent_ref filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment = 0); + +} diff --git a/include/fs/zip/ZipWriter.h b/include/fs/zip/ZipWriter.h new file mode 100644 index 0000000..4668500 --- /dev/null +++ b/include/fs/zip/ZipWriter.h @@ -0,0 +1,77 @@ +#pragma once + +#include "ZipBase.h" +#include "fs/Api.h" +#include +#include +#include +#include +#include + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +class IStreamSource; +class IStreamSink; +class ISink; + +////////////////////////////////////////////////////////////////////////// + +class FS_API ZipWriter : public ZipBase +{ +public: + struct DataWriterPayload + { + CompressionMethod compressionMethod; + uint32_t uncompressedCRC32 = 0; //mandatory!!! + uint64_t uncompressedSize = 0; //optional, will be set to the written size if == 0 + }; + + using DataWriterError = tl::generic_error; + using DataWriterResult = tl::result; + + using DataWriter = tl::function; + + ZipWriter(ISink& sink, size_t fileDataAlignment); + ~ZipWriter() = default; + + tl::result addFile(tl::string name, const DataWriter& dataWriter); + + tl::result addFile(tl::string name, tl::optional lastModificationTime, const DataWriter& dataWriter); + + tl::result addFile(tl::string name, + tl::span comment, + tl::span extraField, + GeneralBitFlags generalBitFlags, + tl::optional lastModificationTime, + const DataWriter& dataWriter); + + void finish(); + +private: + uint64_t computePadding(uint64_t offset, const LocalFileHeader& header) const; + + ISink& m_sink; + size_t m_fileDataAlignment = 0; + + struct CentralDirectoryRecord + { + uint64_t compressedSize = 0; + uint64_t uncompressedSize = 0; + uint64_t localHeaderOffset = 0; + + CentralDirectoryFileHeader header; + tl::string name; + tl::memory_buffer comment; + tl::memory_buffer extraField; + bool zip64 = false; + }; + + tl::vector m_records; + + tl::memory_buffer m_buffer; +}; + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/BufferedFileSource.cpp b/src/BufferedFileSource.cpp new file mode 100644 index 0000000..bbf7eb2 --- /dev/null +++ b/src/BufferedFileSource.cpp @@ -0,0 +1,10 @@ +#include "StdAfx.h" +#include "fs/BufferedFileSource.h" +#include "fs/BufferedSource.inl" + +namespace fs +{ + +template class BufferedSource; + +} diff --git a/src/CRCStream.cpp b/src/CRCStream.cpp new file mode 100644 index 0000000..d1f3864 --- /dev/null +++ b/src/CRCStream.cpp @@ -0,0 +1,255 @@ +#include "StdAfx.h" + +#include "fs/CRCStream.h" + +#include "fs/IFilesystem.h" +#include "fs/IStreamSource.h" +#include "tl/memory_buffer.h" + +extern "C" +{ +#include "zlib-ng.h" +} + +namespace fs +{ + +static constexpr size_t k_defaultBufferSize = 100 * 1024; + +////////////////////////////////////////////////////////////////////////// + +uint32_t computeCRC32(uint32_t crc, tl::span data) +{ + return zng_crc32_z(crc, data.data(), data.size()); +} + +////////////////////////////////////////////////////////////////////////// + + // clang-format off +static constexpr uint64_t s_crc64Table[256] = +{ + UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979), + UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b), + UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6), + UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04), + UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c), + UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe), + UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183), + UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371), + UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8), + UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a), + UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077), + UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285), + UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d), + UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f), + UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02), + UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0), + UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b), + UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489), + UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4), + UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206), + UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e), + UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc), + UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81), + UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73), + UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa), + UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08), + UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75), + UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87), + UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f), + UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d), + UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100), + UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2), + UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416), + UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4), + UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299), + UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b), + UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63), + UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891), + UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec), + UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e), + UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97), + UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965), + UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18), + UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea), + UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2), + UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710), + UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d), + UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f), + UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14), + UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6), + UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b), + UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69), + UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561), + UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793), + UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee), + UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c), + UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495), + UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667), + UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a), + UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8), + UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0), + UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812), + UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f), + UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d), + UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc), + UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e), + UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643), + UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1), + UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9), + UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b), + UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836), + UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4), + UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d), + UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf), + UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2), + UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30), + UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138), + UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca), + UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7), + UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545), + UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce), + UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c), + UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941), + UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3), + UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb), + UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349), + UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734), + UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6), + UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f), + UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd), + UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0), + UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432), + UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a), + UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8), + UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5), + UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47), + UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3), + UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51), + UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c), + UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de), + UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6), + UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124), + UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559), + UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab), + UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222), + UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0), + UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad), + UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f), + UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57), + UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5), + UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8), + UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a), + UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1), + UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053), + UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e), + UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc), + UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4), + UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26), + UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b), + UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9), + UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20), + UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2), + UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf), + UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d), + UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355), + UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7), + UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da), + UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), +}; +// clang-format on + +uint64_t computeCRC64(uint64_t crc, const uint8_t* ptr, size_t size) +{ + while (size >= 4) + { + crc = (crc >> 8) ^ s_crc64Table[(crc ^ ptr[0]) & 0xFF]; + crc = (crc >> 8) ^ s_crc64Table[(crc ^ ptr[1]) & 0xFF]; + crc = (crc >> 8) ^ s_crc64Table[(crc ^ ptr[2]) & 0xFF]; + crc = (crc >> 8) ^ s_crc64Table[(crc ^ ptr[3]) & 0xFF]; + ptr += 4; + size -= 4; + } + + while (size) + { + crc = (crc >> 8) ^ s_crc64Table[(crc ^ ptr[0]) & 0xFF]; + ++ptr; + --size; + } + + return ~crc; +} + +////////////////////////////////////////////////////////////////////////// + +CRC32StreamResult crc32Stream(IStreamSource& source, uint32_t crc, size_t bufferSize) +{ + if (bufferSize == 0) + bufferSize = source.getPreferredBufferSize(); + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + tl::memory_buffer buffer; + buffer.resize_uninitialized(bufferSize); + + do + { + const uint64_t read = source.read(buffer); + if (tl::optional optLastError = source.getLastError(); optLastError.has_value()) + return *optLastError; + + if (read == 0) + return crc; + + crc = zng_crc32_z(crc, buffer.data(), static_cast(read)); + } while (!source.isEOS()); + + return crc; +} + +////////////////////////////////////////////////////////////////////////// + +CRC32StreamResult crc32File(const IFilesystem& filesystem, const AbsPath& filePath, uint32_t crc, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return crc32Stream(*source, crc, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// + +CRC64StreamResult crc64Stream(IStreamSource& source, uint64_t crc, size_t bufferSize) +{ + if (bufferSize == 0) + bufferSize = source.getPreferredBufferSize(); + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + tl::memory_buffer buffer; + buffer.resize_uninitialized(bufferSize); + + do + { + const uint64_t read = source.read(buffer); + if (tl::optional optLastError = source.getLastError(); optLastError.has_value()) + return *optLastError; + + if (read == 0) + return crc; + + crc = computeCRC64(crc, buffer.data(), static_cast(read)); + } while (!source.isEOS()); + + return crc; +} + +////////////////////////////////////////////////////////////////////////// + +CRC64StreamResult crc64File(const IFilesystem& filesystem, const AbsPath& filePath, uint64_t crc, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return crc64Stream(*source, crc, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/CopyStream.cpp b/src/CopyStream.cpp new file mode 100644 index 0000000..060b966 --- /dev/null +++ b/src/CopyStream.cpp @@ -0,0 +1,85 @@ +#include "StdAfx.h" +#include "fs/CopyStream.h" +#include "fs/IStreamSink.h" +#include "fs/IStreamSource.h" +#include + +#include "fs/IFilesystem.h" + +namespace fs +{ + +static constexpr size_t k_defaultBufferSize = 100 * 1024; + +CopyStreamResult copyStream(IStreamSink& sink, IStreamSource& source, size_t bufferSize) +{ + return copyStream(sink, source, nullptr, bufferSize); +} + +CopyStreamResult copyStream(IStreamSink& sink, IStreamSource& source, const ProcessDataCallback& dataCallback, size_t bufferSize) +{ + if (bufferSize == 0) + bufferSize = source.getPreferredBufferSize(); + + if (bufferSize == 0) + bufferSize = sink.getPreferredBufferSize(); + + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + tl::memory_buffer buffer; + buffer.resize_uninitialized(bufferSize); + + uint64_t size = 0; + do + { + const uint64_t read = source.read(buffer); + tl::optional optLastError = source.getLastError(); + if (optLastError.has_value()) + return *optLastError; + + if (read == 0) + return size; + + if (dataCallback) + dataCallback({ buffer.data(), static_cast(read) }); + + const uint64_t written = sink.write({ buffer.data(), static_cast(read) }); + if (written != read) + return tl::make_error(ErrorCode::SystemError, "Failed to write {} bytes. Only {} went through", read, written); + + size += written; + + optLastError = sink.getLastError(); + if (optLastError.has_value()) + return *optLastError; + } while (!source.isEOS()); + + return size; +} + +////////////////////////////////////////////////////////////////////////// + +CopyStreamResult copyFile(IStreamSink& sink, const IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return copyStream(sink, *source, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// + +CopyStreamResult copyFile(IStreamSink& sink, const IFilesystem& filesystem, const AbsPath& filePath, const ProcessDataCallback& dataCallback, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return copyStream(sink, *source, dataCallback, bufferSize); +} + +CopyStreamResult copyFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, const IFilesystem& srcFilesystem, const AbsPath& srcFilePath, size_t bufferSize) +{ + OUTCOME_TRY(const auto sink, dstFilesystem.openStreamSink(dstFilePath, fs::Mode::CreateOrOpenAndClear)); + OUTCOME_TRY(const auto source, srcFilesystem.openStreamSource(srcFilePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return copyStream(*sink, *source, nullptr, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/CustomFilesystem.cpp b/src/CustomFilesystem.cpp new file mode 100644 index 0000000..94b4e13 --- /dev/null +++ b/src/CustomFilesystem.cpp @@ -0,0 +1,609 @@ +#include "StdAfx.h" +#include "fs/CustomFilesystem.h" +#include "fs/WritableFolderPack.h" +#include "tl/assert.h" +//#include "tl/algorithm/find.h" +#include "fs/CopyStream.h" +#include "fs/NativeFilesystem.h" +#include "tl/unordered_set.h" +#include "tl/algorithm/find.h" +#include + +////////////////////////////////////////////////////////////////////////// + +template class tl::abs_path>; +template class tl::rel_path>; + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +namespace +{ +tl::atomic s_lastId(0); +} + +////////////////////////////////////////////////////////////////////////// + +tl::result CustomFilesystem::mountFront(AbsPath mountPoint, tl::unique_ref pack) +{ + return mount(std::move(mountPoint), std::move(pack), MountPolicy::Front); +} + +////////////////////////////////////////////////////////////////////////// + +tl::result CustomFilesystem::mountBack(AbsPath mountPoint, tl::unique_ref pack) +{ + return mount(std::move(mountPoint), std::move(pack), MountPolicy::Back); +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator> CustomFilesystem::unmountAll() +{ + tl::fixed_vector, 32> packs; + packs.reserve(m_packsData.size()); + for (PackData& pd: m_packsData) + packs.push_back(std::move(pd.pack)); + + m_packsData.clear(); + + for (auto& pack: packs) + co_yield std::move(pack); +} + +////////////////////////////////////////////////////////////////////////// + +tl::unique_ptr CustomFilesystem::unmount(PackId packId) +{ + if (const auto it = tl::find_if(m_packsData, [packId](const PackData& p) { return p.id == packId; }); it != m_packsData.end()) + { + auto p = std::move(it->pack); //move it before erasing + m_packsData.erase(it); + return std::move(p); + } + return nullptr; +} + +////////////////////////////////////////////////////////////////////////// + +OpenSourceResult CustomFilesystem::openSource(const AbsPath& path, SourceFlags flags) const +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + const ExistsResult existsResult = pd.pack->exists(filePackPath); + if (existsResult == tl::success() && existsResult.value() == false) + continue; + + tl::result, Error> result = pd.pack->openSource(filePackPath, flags); + if (result.has_value()) + return std::move(result.value()); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found", path); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSourceResult CustomFilesystem::openStreamSource(const AbsPath& path, SourceFlags flags) const +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + const ExistsResult existsResult = pd.pack->exists(filePackPath); + if (existsResult == tl::success() && existsResult.value() == false) + continue; + + tl::result, Error> result = pd.pack->openStreamSource(filePackPath, flags); + if (result.has_value()) + return std::move(result.value()); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found", path); +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSourceResult CustomFilesystem::openMapSource(const AbsPath& path, MapView mapView, SourceFlags flags) const +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + const ExistsResult existsResult = pd.pack->exists(filePackPath); + if (existsResult == tl::success() && existsResult.value() == false) + continue; + + tl::result, Error> result = pd.pack->openMapSource(filePackPath, mapView, flags); + if (result.has_value()) + return std::move(result.value()); + if (result.error().code != ErrorCode::NotFound) + return result.error(); + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found", path); +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSinkResult CustomFilesystem::openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags) +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + OpenMapSinkResult result = pack->openMapSink(filePackPath, mode, size, flags); + if (result.has_value()) + return std::move(result.value()); + if (result.error().code != ErrorCode::NotFound) + return result; + } + else + return tl::make_error(ErrorCode::NotAllowed, "The filesystem is read only"); + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found", path); +} + +////////////////////////////////////////////////////////////////////////// + +OpenSinkResult CustomFilesystem::openSink(const AbsPath& path, Mode mode, SinkFlags flags) +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + OpenSinkResult result = pack->openSink(filePackPath, mode, flags); + if (result.has_value()) + return std::move(result.value()); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found or all mounted packs are read only", path); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSinkResult CustomFilesystem::openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags) +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + OpenStreamSinkResult result = pack->openStreamSink(filePackPath, mode, flags); + if (result.has_value()) + return std::move(result.value()); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found or all mounted packs are read only", path); +} + +////////////////////////////////////////////////////////////////////////// + +ExistsResult CustomFilesystem::exists(const AbsPath& path) const +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + if (filePackPath.empty()) //This is the exact mount point of the pack, which means this folder exists + return true; + + auto result = pd.pack->exists(filePackPath); + if (result.has_value() && result.value() == true) + return true; + + if (result.has_error()) + { + //exists should not return not found + TL_ASSERT(result.error().code != ErrorCode::NotFound); + return result; + } + } + } + + return false; +} + +////////////////////////////////////////////////////////////////////////// + +IsFileResult CustomFilesystem::isFile(const AbsPath& path) const +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + if (filePackPath.empty()) //This is the exact mount point of the pack, which means it is not a file + return false; + + auto result = pd.pack->isFile(filePackPath); + if (result.has_value()) + return result.value(); + if (result.error().code != Error::code_t::NotFound) + return result; + } + } + + return tl::make_error(Error::code_t::NotFound, "Path '{}' not found in the filesystem", path); +} + +////////////////////////////////////////////////////////////////////////// + +RenameResult CustomFilesystem::rename(const AbsPath& path, const AbsPath& newPath) +{ + AbsPath fromPackPath; + AbsPath toPackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(fromPackPath, path, pd.mountPoint); + convertToPackPath(toPackPath, newPath, pd.mountPoint); + + auto result = pack->rename(fromPackPath, toPackPath); + if (result.has_value()) + return tl::success(); + if (result.error().code != Error::code_t::NotFound) + return result; + } + } + } + + return tl::make_error(Error::code_t::NotFound, "Path '{}' was not found or all mounted packs are read only", path); +} + +////////////////////////////////////////////////////////////////////////// + +CopyResult CustomFilesystem::copy(const AbsPath& path, const AbsPath& newPath) +{ + OUTCOME_TRY(const auto source, openStreamSource(path, SourceFlags(SourceFlag::SequentialHint, SourceFlag::Uncached))); + OUTCOME_TRY(const auto destination, openStreamSink(newPath, Mode::CreateOrOpenAndClear)); + + CopyStreamResult copyResult = copyStream(*destination, *source); + if (copyResult.has_error()) + return copyResult.error(); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +MakeHardLinkResult CustomFilesystem::makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) +{ + OUTCOME_TRY(const auto sp, convertToNativePath(sourcePath)); + OUTCOME_TRY(const auto lp, convertToNativePath(linkPath)); + return native.makeHardLink(sp, lp); +} + +////////////////////////////////////////////////////////////////////////// + +MakeSoftLinkResult CustomFilesystem::makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) +{ + OUTCOME_TRY(const auto sp, convertToNativePath(sourcePath)); + OUTCOME_TRY(const auto lp, convertToNativePath(linkPath)); + return native.makeSymLink(sp, lp); +} + +////////////////////////////////////////////////////////////////////////// + +IsFolderResult CustomFilesystem::isFolder(const AbsPath& path) const +{ + AbsPath folderPackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(folderPackPath, path, pd.mountPoint); + if (folderPackPath.empty()) //This is the exact mount point of the pack, which means it is a folder + return true; + + auto result = pd.pack->isFolder(folderPackPath); + if (result.has_value()) + return result.value(); + if (result.error().code != Error::code_t::NotFound) + return result; + } + } + + return tl::make_error(Error::code_t::NotFound, "Path '{}' not found in the filesystem", path); +} + +////////////////////////////////////////////////////////////////////////// + +MakeFolderResult CustomFilesystem::makeFolder(const AbsPath& path) +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + MakeFolderResult result = pack->makeFolder(filePackPath); + if (result.has_value()) + return result; + if (result.error().code != ErrorCode::NotFound) + return result; + } + else + return tl::make_error(ErrorCode::NotAllowed, "The filesystem is read only"); + } + } + + return tl::make_error(Error::code_t::SystemError, "Not implemented"); +} + +////////////////////////////////////////////////////////////////////////// + +RemoveResult CustomFilesystem::remove(const AbsPath& path) +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + auto result = pack->remove(filePackPath); + if (result.has_value()) + return tl::success(); + if (result.error().code != Error::code_t::NotFound) + return result; + } + } + } + + return tl::make_error(ErrorCode::NotFound, "Path '{}' was not found or all mounted packs are read only", path); +} + +////////////////////////////////////////////////////////////////////////// + +RemoveRecursivelyResult CustomFilesystem::removeRecursively(const AbsPath& path) +{ + for (const EnumerateEntry& ee : enumerate(path)) + { + AbsPath p = path + ee.path; + if (ee.isFolder) + { + OUTCOME_TRY(removeRecursively(p)); + } + else + { + OUTCOME_TRY(remove(p)); + } + } + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +ConvertToNativePathResult CustomFilesystem::convertToNativePath(const AbsPath& path) const +{ + AbsPath folderPackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(folderPackPath, path, pd.mountPoint); + auto result = pd.pack->convertToNativePath(folderPackPath); + if (result.has_value()) + return result.value(); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + + return tl::make_error(Error::code_t::NotFound, "Cannot convert '{}' to a native path", path); +} + +////////////////////////////////////////////////////////////////////////// + +SetWriteTimeResult CustomFilesystem::setWriteTime(const AbsPath& path, time_t time) +{ + AbsPath filePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + if (const tl::lent_ptr pack = tl::dynamic_lent_cast(pd.pack)) + { + convertToPackPath(filePackPath, path, pd.mountPoint); + + SetWriteTimeResult result = pack->setWriteTime(filePackPath, time); + if (result.has_value()) + return tl::success(); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + } + + return tl::make_error(Error::code_t::NotFound, "Path '{}' was not found or all mounted packs are read only", path); +} + +////////////////////////////////////////////////////////////////////////// + +GetStatResult CustomFilesystem::getStat(const AbsPath& path) const +{ + AbsPath folderPackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(folderPackPath, path, pd.mountPoint); + auto result = pd.pack->getStat(folderPackPath); + if (result.has_value()) + return result.value(); + if (result.error().code != ErrorCode::NotFound) + return result; + } + } + + return tl::make_error(Error::code_t::NotFound, "File {} not found", path); +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator CustomFilesystem::enumerate(const AbsPath& path) const +{ + AbsPath basePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(basePackPath, path, pd.mountPoint); + + //it's possible that even if the mount point is + const IsFolderResult isFolderResult = pd.pack->isFolder(basePackPath); + if (isFolderResult.has_error() || isFolderResult.value() == false) + continue; + + for (EnumerateEntry ee : pd.pack->enumerate(basePackPath)) + co_yield std::move(ee); + } + } +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator CustomFilesystem::enumerateRecursively(const AbsPath& path) const +{ + tl::unordered_set checkedFiles; + + AbsPath basePackPath; + for (const auto& pd : m_packsData) + { + if (pd.mountPoint.is_prefix_of(path)) + { + convertToPackPath(basePackPath, path, pd.mountPoint); + + if (!basePackPath.empty()) + { + const IsFolderResult isFolderResult = pd.pack->isFolder(basePackPath); + if (isFolderResult.has_error() || isFolderResult.value() == false) + continue; + } + + for (EnumerateEntry ee: pd.pack->enumerateRecursively(basePackPath)) + { + if (checkedFiles.insert(ee.path).second) + co_yield std::move(ee); + } + } + else if (path.is_prefix_of(pd.mountPoint)) + { + const RelPath parentPath = path.path_to(pd.mountPoint); + + for (EnumerateEntry ee : pd.pack->enumerateRecursively(AbsPath(PosixRootTag::value()))) + { + ee.path = parentPath + ee.path; + if (checkedFiles.insert(ee.path).second) + co_yield std::move(ee); + } + } + } +} + +////////////////////////////////////////////////////////////////////////// + +void CustomFilesystem::convertToPackPath(AbsPath& packPath, const AbsPath& path, const AbsPath& mountPoint) const +{ + RelPath basePath = mountPoint.path_to(path); + + packPath = PosixRootTag::value(); + packPath.reserve(basePath.size() + 1); + for (tl::string& element : basePath) + packPath.push_back(std::move(element)); +} + +////////////////////////////////////////////////////////////////////////// + +tl::result CustomFilesystem::mount(AbsPath mountPoint, tl::unique_ref pack, MountPolicy policy) +{ + PackData data(std::move(pack)); + + if (!mountPoint.is_valid()) + return tl::make_generic_error("Invalid mount point"); + + auto isFileResult = isFile(mountPoint); + if (isFileResult.has_value() && isFileResult.value() == true) + return tl::make_generic_error("Invalid shadowing (mount point shadowing a file)"); + + const PackId id(++s_lastId); + data.id = id; + data.mountPoint = std::move(mountPoint); + + switch (policy) + { + case MountPolicy::Front: + m_packsData.insert(m_packsData.begin(), std::move(data)); + break; + case MountPolicy::Back: + m_packsData.push_back(std::move(data)); + break; + default: + TL_FAIL(); + break; + } + + return id; +} + +////////////////////////////////////////////////////////////////////////// +} + diff --git a/src/DeflateStreamSink.cpp b/src/DeflateStreamSink.cpp new file mode 100644 index 0000000..f72a21e --- /dev/null +++ b/src/DeflateStreamSink.cpp @@ -0,0 +1,205 @@ +#include "StdAfx.h" +#include "fs/DeflateStreamSink.h" +#include "tl/narrow_cast.h" + +extern "C" +{ +#include "zlib-ng.h" +} + +namespace fs +{ + +static constexpr size_t k_preferredBufferSize = 100 * 1024; +static constexpr size_t k_outBufferSize = 100 * 1024; + +struct DeflateStreamSink::ZLib +{ + zng_stream streamer; +}; + +////////////////////////////////////////////////////////////////////////// + +DeflateStreamSink::DeflateStreamSink(tl::unique_ref sink) + : m_ownedSink(std::move(sink)) + , m_sink(*m_ownedSink) +{ + +} + +////////////////////////////////////////////////////////////////////////// + +DeflateStreamSink::DeflateStreamSink(IStreamSink& sink) + : m_sink(sink) +{ + +} + +////////////////////////////////////////////////////////////////////////// + +DeflateStreamSink::~DeflateStreamSink() +{ + if (!m_isFinished) + flush(true); + + if (m_zlib) + (void)zng_deflateEnd(&m_zlib->streamer); +} + +////////////////////////////////////////////////////////////////////////// + +void DeflateStreamSink::setCompressionLevel(uint8_t level) +{ + m_level = tl::min(level, 9u); +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional DeflateStreamSink::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +void DeflateStreamSink::initializeDeflate() +{ + m_buffer.resize_uninitialized(getPreferredBufferSize()); + m_bufferOffset = 0; + m_outBuffer.resize_uninitialized(m_sink.getPreferredBufferSize() > 0 ? m_sink.getPreferredBufferSize() : k_outBufferSize); + + m_zlib = tl::make_unique(); + m_zlib->streamer.zalloc = nullptr; + m_zlib->streamer.zfree = nullptr; + m_zlib->streamer.opaque = nullptr; + const int ret = zng_deflateInit2(&m_zlib->streamer, m_level, Z_DEFLATED, -MAX_WBITS, 9, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) + { + (void)zng_deflateEnd(&m_zlib->streamer); + m_zlib = nullptr; + } +} + +////////////////////////////////////////////////////////////////////////// + +size_t DeflateStreamSink::write(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); //the previous file access generated an error + return 0; + } + + if (data.empty()) + return 0; + + if (!data.data()) + { + m_optLastError = tl::make_error(ErrorCode::InvalidArgument, "Null buffer."); + TL_FAIL("{}", m_optLastError); + return 0; + } + + size_t left = data.size(); + const uint8_t* src = data.data(); + + if (!m_zlib) + initializeDeflate(); + + if (!m_zlib) + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "Failed to initialize the compressor"); + TL_FAIL("{}", m_optLastError); + return 0; + } + + while (left > 0) + { + if (const size_t s = tl::min(left, m_buffer.size() - m_bufferOffset)) + { + memcpy(m_buffer.data() + m_bufferOffset, src, s); + src += s; + left -= s; + m_bufferOffset += s; + m_size += s; + } + + if (!flush(false)) + return 0; + } + + return data.size(); +} + +////////////////////////////////////////////////////////////////////////// + +bool DeflateStreamSink::flush(bool finalize) +{ + if (!m_zlib) + return true; + + if (finalize || (m_bufferOffset > 0 && m_bufferOffset >= m_buffer.size())) + { + m_zlib->streamer.avail_in = tl::narrow(m_bufferOffset); + m_zlib->streamer.next_in = m_buffer.data(); + + do + { + m_zlib->streamer.avail_out = tl::narrow(m_outBuffer.size()); + m_zlib->streamer.next_out = m_outBuffer.data(); + const int ret = zng_deflate(&m_zlib->streamer, finalize ? Z_FINISH : Z_NO_FLUSH); + TL_ASSERT(ret != Z_STREAM_ERROR); + (void)ret; + const size_t available = m_outBuffer.size() - m_zlib->streamer.avail_out; + const uint64_t written = m_sink.write({ m_outBuffer.data(), available }); + tl::optional optLastError = m_sink.getLastError(); + if (written != available || optLastError.has_value()) + { + m_buffer.clear(); + m_bufferOffset = 0; + m_optLastError = tl::make_error(ErrorCode::SystemError, "Subsink failed while writting data"); + TL_FAIL("{}", m_optLastError); + return false; + } + } while (m_zlib->streamer.avail_out == 0); + + m_bufferOffset = 0; + } + return true; +} + +////////////////////////////////////////////////////////////////////////// + +void DeflateStreamSink::finish() +{ + TL_ASSERT(!m_isFinished); + m_isFinished = true; + flush(true); +} + +////////////////////////////////////////////////////////////////////////// + +tl::unique_ref DeflateStreamSink::finishAndExtract() +{ + finish(); + return tl::promote(std::move(m_ownedSink)); +} + +////////////////////////////////////////////////////////////////////////// + +size_t DeflateStreamSink::getPreferredBufferSize() const +{ + return m_sink.getPreferredBufferSize() > 0 ? m_sink.getPreferredBufferSize() : k_preferredBufferSize; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t DeflateStreamSink::getSize() const +{ + return m_size; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/DeflateStreamSource.cpp b/src/DeflateStreamSource.cpp new file mode 100644 index 0000000..3281ba6 --- /dev/null +++ b/src/DeflateStreamSource.cpp @@ -0,0 +1,182 @@ +#include "StdAfx.h" +#include "fs/DeflateStreamSource.h" + +#include "tl/narrow_cast.h" + +extern "C" +{ +#include "zlib-ng.h" +} + +namespace fs +{ + +static constexpr size_t k_preferredBufferSize = 100 * 1024; + +struct DeflateStreamSource::ZLib +{ + zng_stream streamer; + bool needsData = true; +}; + +////////////////////////////////////////////////////////////////////////// + +DeflateStreamSource::DeflateStreamSource(tl::unique_ref source) + : m_ownedSource(std::move(source)) + , m_source(*m_ownedSource) +{ + +} + +////////////////////////////////////////////////////////////////////////// + +DeflateStreamSource::DeflateStreamSource(IStreamSource& source) + : m_source(source) +{ + +} + +////////////////////////////////////////////////////////////////////////// + +DeflateStreamSource::~DeflateStreamSource() +{ + if (m_zlib) + (void)zng_inflateEnd(&m_zlib->streamer); +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional DeflateStreamSource::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +void DeflateStreamSource::initializeDeflate() +{ + m_buffer.resize_uninitialized(getPreferredBufferSize()); + + m_zlib = tl::make_unique(); + m_zlib->streamer.zalloc = nullptr; + m_zlib->streamer.zfree = nullptr; + m_zlib->streamer.opaque = nullptr; + m_zlib->streamer.next_in = nullptr; + m_zlib->streamer.avail_in = 0; + const int ret = zng_inflateInit2(&m_zlib->streamer, -MAX_WBITS); + if (ret != Z_OK) + { + (void)zng_inflateEnd(&m_zlib->streamer); + m_zlib = nullptr; + } +} + +////////////////////////////////////////////////////////////////////////// + +size_t DeflateStreamSource::read(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); //the previous file access generated an error + return 0; + } + + if (data.empty()) + return 0; + + if (!m_zlib) + initializeDeflate(); + + if (!m_zlib) + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "Failed to initialize the compressor"); + TL_FAIL("{}", m_optLastError); + return 0; + } + + size_t left = data.size(); + uint8_t* dst = data.data(); + + do + { + if (m_zlib->needsData) + { + //this end of source is valid, and it means end of compressed source. + if (m_source.isEOS()) + return data.size() - left; + + m_buffer.resize_uninitialized(getPreferredBufferSize()); + + const uint64_t read = m_source.read(m_buffer); + if (tl::optional optLastError = m_source.getLastError(); optLastError.has_value()) + { + m_buffer.clear(); + m_optLastError = tl::make_error(ErrorCode::SystemError, "Unexpected end of encrypted source"); + TL_FAIL("{}", m_optLastError); + return 0; + } + m_buffer.resize_uninitialized(static_cast(read)); + + m_zlib->streamer.avail_in = tl::narrow(m_buffer.size()); + m_zlib->streamer.next_in = m_buffer.data(); + } + + m_zlib->streamer.avail_out = tl::narrow(left); + m_zlib->streamer.next_out = reinterpret_cast(dst); + const int ret = zng_inflate(&m_zlib->streamer, Z_NO_FLUSH); + TL_ASSERT(ret != Z_STREAM_ERROR); + switch (ret) + { + case Z_OK: break; //all good + case Z_STREAM_END: break; //all good + case Z_NEED_DICT: + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "deflate dictionary error ({})", ret); + TL_FAIL("{}", m_optLastError); + return 0; + } + case Z_DATA_ERROR: + case Z_MEM_ERROR: + default: + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "deflate data error ({})", ret); + TL_FAIL("{}", m_optLastError); + return 0; + } + } + + const size_t read = left - m_zlib->streamer.avail_out; + left -= read; + dst += read; + + m_zlib->needsData = m_zlib->streamer.avail_out != 0; + } while (left > 0); + + return data.size(); +} + +////////////////////////////////////////////////////////////////////////// + +tl::unique_ref DeflateStreamSource::extract() +{ + return tl::promote(std::move(m_ownedSource)); +} + +////////////////////////////////////////////////////////////////////////// + +bool DeflateStreamSource::isEOS() const +{ + return m_source.isEOS() && (!m_zlib || m_zlib->needsData); +} + +////////////////////////////////////////////////////////////////////////// + +size_t DeflateStreamSource::getPreferredBufferSize() const +{ + return m_source.getPreferredBufferSize() > 0 ? m_source.getPreferredBufferSize() : k_preferredBufferSize; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/FileMapSink.cpp b/src/FileMapSink.cpp new file mode 100644 index 0000000..1d2ea9f --- /dev/null +++ b/src/FileMapSink.cpp @@ -0,0 +1,183 @@ +#include "StdAfx.h" +#include "fs/FileMapSink.h" +#include + +#include "fs/NativeFilesystem.h" +#include "tl/narrow_cast.h" + +namespace fs +{ +//////////////////////////////////////////////////////////////////////////// + +struct FileMapSink::Impl +{ + mio::ummap_sink sink; +}; + +FileMapSink::FileMapSink(const AbsPath& filepath, + size_t size, + Mode mode /*= Mode::CreateOrOpenAndClear*/, + Flags flags /*= Flags()*/) +{ + //create and write file + { + auto openResult = native.openSink(filepath, mode); + if (openResult.has_value()) + { + const auto fs = std::move(openResult.value()); + if (size > fs->getSize()) + { + fs->seekBeg(size - 1); + constexpr uint8_t buf[1] = { 0 }; + fs->write( { buf, 1 }); + } + } + } + + std::error_code error; + mio::ummap_sink mmap; + mmap.map(filepath.std_str(), 0, size, error); + if (error) + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "{}", error.message()); + return; + } + + m_impl = tl::make_unique(std::move(mmap)); +} + +////////////////////////////////////////////////////////////////////////// + +FileMapSink::~FileMapSink() +{ + if (m_impl) + { + std::error_code error; + m_impl->sink.sync(error); + } +} + +////////////////////////////////////////////////////////////////////////// + +bool FileMapSink::isValid() const +{ + return m_impl != nullptr; +} + +//////////////////////////////////////////////////////////////////////////// + +size_t FileMapSink::write(tl::span data) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + const size_t sz = tl::min(data.size(), m_impl->sink.size() - tl::narrow(m_offset)); + if (sz > 0) + { + memcpy(m_impl->sink.data() + m_offset, data.data(), sz); + m_offset += sz; + } + + return sz; +} + +//////////////////////////////////////////////////////////////////////////// + +void FileMapSink::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (m_offset != offset) + m_offset = tl::min(offset, static_cast(m_impl->sink.size())); +} + +//////////////////////////////////////////////////////////////////////////// + +void FileMapSink::seekRel(int64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +//////////////////////////////////////////////////////////////////////////// + +uint64_t FileMapSink::tell() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +//////////////////////////////////////////////////////////////////////////// + +uint64_t FileMapSink::getSize() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return m_impl->sink.size(); +} + +//////////////////////////////////////////////////////////////////////////// + +tl::span FileMapSink::map(size_t size) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return {}; + } + + const size_t bytesToMap = tl::min(m_impl->sink.size() - tl::narrow(m_offset), size); + if (bytesToMap == 0) + return {}; + + const tl::span memView(m_impl->sink.data() + m_offset, bytesToMap); + seekRel(bytesToMap); + return memView; +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional FileMapSink::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +size_t FileMapSink::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/FileMapSource.cpp b/src/FileMapSource.cpp new file mode 100644 index 0000000..3dbda01 --- /dev/null +++ b/src/FileMapSource.cpp @@ -0,0 +1,190 @@ +#include "StdAfx.h" +#include "fs/FileMapSource.h" + +#include "Logger.h" +#include + +#include "tl/narrow_cast.h" +#include "mio/mmap.hpp" + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +struct FileMapSource::Impl +{ + mio::ummap_source source; +}; + +FileMapSource::FileMapSource(const AbsPath& filepath) +{ + std::error_code error; + mio::ummap_source mmap; + mmap.map(filepath.std_str(), error); + if (error) + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "{}", error.message()); + return; + } + + m_impl = tl::make_unique(std::move(mmap)); +} + +////////////////////////////////////////////////////////////////////////// + +FileMapSource::FileMapSource(const AbsPath& filepath, uint64_t start, size_t size) +{ + std::error_code error; + mio::ummap_source mmap; + mmap.map(filepath.std_str(), start, size, error); + if (error) + { + m_optLastError = tl::make_error(ErrorCode::SystemError, "{}", error.message()); + return; + } + + m_impl = tl::make_unique(std::move(mmap)); +} + +////////////////////////////////////////////////////////////////////////// + +FileMapSource::~FileMapSource() +{ + m_impl = {}; +} + +////////////////////////////////////////////////////////////////////////// + +bool FileMapSource::isValid() const +{ + return m_impl != nullptr; +} + +//////////////////////////////////////////////////////////////////////////// + +size_t FileMapSource::read(tl::span data) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + const uint64_t sz = tl::min(static_cast(data.size()), m_impl->source.size() - m_offset); + if (sz > 0) + { + std::memcpy(data.data(), m_impl->source.data() + m_offset, tl::narrow(sz)); + m_offset += sz; + } + + return static_cast(sz); +} + +//////////////////////////////////////////////////////////////////////////// + +void FileMapSource::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + m_offset = tl::min(offset, getSize()); +} + +//////////////////////////////////////////////////////////////////////////// + +void FileMapSource::seekRel(int64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +//////////////////////////////////////////////////////////////////////////// + +uint64_t FileMapSource::tell() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +//////////////////////////////////////////////////////////////////////////// + +uint64_t FileMapSource::getSize() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return m_impl->source.size(); +} + +//////////////////////////////////////////////////////////////////////////// + +tl::span FileMapSource::map(size_t size) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return {}; + } + + const uint64_t remainingBytes = getSize() - m_offset; + const uint64_t bytesToMap = tl::min(remainingBytes, static_cast(size)); + if (bytesToMap == 0) + return {}; + + const tl::span memView(m_impl->source.data() + m_offset, tl::narrow(bytesToMap)); + seekRel(bytesToMap); + return memView; +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional FileMapSource::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +//////////////////////////////////////////////////////////////////////////// + +bool FileMapSource::isEOS() const +{ + if (!isValid()) + return true; + + return m_offset >= getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t FileMapSource::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/FileSink.cpp b/src/FileSink.cpp new file mode 100644 index 0000000..3cdf10b --- /dev/null +++ b/src/FileSink.cpp @@ -0,0 +1,259 @@ +#include "StdAfx.h" + +#include "fs/FileSink.h" + +//#include "fs/FileIncludes.h" +#include "fs/NativeUtils.h" +#include "tl/crash.h" +#include "tl/enum.h" + +#include +#include "fast_io.h" + +#include "tl/narrow_cast.h" + +////////////////////////////////////////////////////////////////////////// + +using FH = fast_io::native_file; + +template +FH& toFH(Arena& arena) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + return *reinterpret_cast(arena.data()); +} +template +const FH& toFH(const Arena& arena) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + return *reinterpret_cast(arena.data()); +} +template +FH& constructFH(Arena& arena, FH&& f) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + return *new (arena.data()) FH(std::move(f)); +} +template +void destructFH(Arena& arena) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + toFH(arena).~FH(); +} + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +FileSink::FileSink(const AbsPath& filepath, Mode mode, Flags flags) noexcept +{ + fast_io::open_mode fmode = fast_io::open_mode::out; + + switch (mode) + { + case Mode::CreateOrOpen: + fmode |= fast_io::open_mode::creat; + // fall through + case Mode::Open: + break; + case Mode::CreateOrOpenAndClear: + fmode |= fast_io::open_mode::creat; + // fall through + case Mode::OpenAndClear: + fmode |= fast_io::open_mode::trunc; + break; + case Mode::CreateIfNew: + fmode |= fast_io::open_mode::creat; + fmode |= fast_io::open_mode::excl; + break; + default: + TL_CRASH("Unknown sink Mode: {}", (int)mode); + } + + auto& fh = constructFH(m_arena, fast_io::native_file(filepath.std_str(), fmode, fast_io::perms::owner_read | fast_io::perms::owner_write)); + m_isConstructed = true; +} + +////////////////////////////////////////////////////////////////////////// + +FileSink::FileSink(FileSink&& other) noexcept + : m_isConstructed(other.m_isConstructed) + , m_arena(other.m_arena) + , m_offset(other.m_offset) + , m_optLastError(std::move(other.m_optLastError)) +{ + other.m_isConstructed = false; + other.m_offset = 0; +} + +////////////////////////////////////////////////////////////////////////// + +FileSink& FileSink::operator=(FileSink&& other) noexcept +{ + close(); + tl::swap(m_isConstructed, other.m_isConstructed); + tl::swap(m_arena, other.m_arena); + tl::swap(m_offset, other.m_offset); + m_optLastError = std::move(other.m_optLastError); + + return *this; +} + +////////////////////////////////////////////////////////////////////////// + +bool FileSink::isValid() const noexcept +{ + if (m_isConstructed) + { + auto& fh = toFH(m_arena); + return fh.handle != nullptr; + } + return false; +} + +////////////////////////////////////////////////////////////////////////// + +FileSink::~FileSink() noexcept +{ + close(); + if (m_isConstructed) + destructFH(m_arena); +} + +////////////////////////////////////////////////////////////////////////// + +void FileSink::close() noexcept +{ + m_offset = 0; + m_optLastError = tl::nullopt; + + if (isValid()) + { + auto& fh = toFH(m_arena); + fh.close(); + } +} + +////////////////////////////////////////////////////////////////////////// + +size_t FileSink::write(tl::span data) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + if (data.empty()) + return 0; + + const auto it = fast_io::write(toFH(m_arena), data.begin(), data.end()); + const int64_t sizeWritten = std::distance(data.begin(), it); + + if (sizeWritten < 0) + { + int err = errno; + m_optLastError = tl::make_error(ErrorCode::SystemError, "Failed to write to file: ({}): {}", err, strerror(err)); + TL_FAIL("{}", m_optLastError); + return 0; + } + m_offset += static_cast(sizeWritten); + + return sizeWritten; +} + +////////////////////////////////////////////////////////////////////////// + +void FileSink::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (m_offset != offset) + m_offset = fast_io::seek(toFH(m_arena), offset, fast_io::seekdir::beg); +} + +////////////////////////////////////////////////////////////////////////// + +void FileSink::seekRel(int64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t FileSink::tell() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t FileSink::getSize() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return fast_io::status(toFH(m_arena)).size; +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional FileSink::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +size_t FileSink::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +bool FileSink::resize(uint64_t newSize) noexcept +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return false; + } + + fast_io::truncate(toFH(m_arena), newSize); + fast_io::seek(toFH(m_arena), newSize, fast_io::seekdir::beg); + m_offset = newSize; + + return true; +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/FileSource.cpp b/src/FileSource.cpp new file mode 100644 index 0000000..dfe9b6f --- /dev/null +++ b/src/FileSource.cpp @@ -0,0 +1,267 @@ +#include "StdAfx.h" + +//#include "fs/FileIncludes.h" +#include "fs/FileSource.h" +#include "fs/NativeUtils.h" + +#include + +#include "tl/narrow_cast.h" + +#include "fast_io.h" + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +namespace native_utils +{ +extern int validateCaseSensitiveFilename(const AbsPath& filepath, int fd); +#if defined (TL_PLATFORM_WINDOWS_FAMILY) +extern int validateCaseSensitiveFilename(const AbsPath& filepath, HANDLE handle); +#endif +} + +using FH = fast_io::native_file; + +template +FH& toFH(Arena& arena) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + return *reinterpret_cast(arena.data()); +} +template +const FH& toFH(const Arena& arena) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + return *reinterpret_cast(arena.data()); +} +template +FH& constructFH(Arena& arena, FH&& f) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + return *new (arena.data()) FH(std::move(f)); +} +template +void destructFH(Arena& arena) +{ + static_assert(sizeof(arena) >= sizeof(FH)); + toFH(arena).~FH(); +} + +////////////////////////////////////////////////////////////////////////// + +FileSource::FileSource(FileSource&& other) noexcept +{ + swap(other); +} + +////////////////////////////////////////////////////////////////////////// + +FileSource& FileSource::operator=(FileSource&& other) noexcept +{ + if (this != &other) + { + FileSource tmp; + tmp.swap(other); + swap(tmp); + } + + return *this; +} + +////////////////////////////////////////////////////////////////////////// + +FileSource::FileSource(const AbsPath& filepath) noexcept + : FileSource(filepath, Flags()) +{ +} + +////////////////////////////////////////////////////////////////////////// + +FileSource::FileSource(const AbsPath& filepath, Flags flags) noexcept +{ + const tl::string pathStr = filepath.str(); + + fast_io::open_mode mode = fast_io::open_mode::in | fast_io::open_mode::shared_delete; + + if (flags.test(Flag::SequentialHint)) + //mode |= fast_io::open_mode:; + ; + else if (flags.test(Flag::RandomHint)) + mode |= fast_io::open_mode::random_access; + + auto& fh = constructFH(m_arena, fast_io::native_file(pathStr, mode)); + m_isConstructed = true; +} + +////////////////////////////////////////////////////////////////////////// + +bool FileSource::isValid() const noexcept +{ + if (m_isConstructed) + { + auto& fh = toFH(m_arena); + return fh.handle != nullptr; + } + return false; +} + +////////////////////////////////////////////////////////////////////////// + +FileSource::~FileSource() noexcept +{ + close(); + if (m_isConstructed) + destructFH(m_arena); +} + +////////////////////////////////////////////////////////////////////////// + +void FileSource::close() noexcept +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + if (isValid()) + { + auto& fh = toFH(m_arena); + fh.close(); + } +} + +////////////////////////////////////////////////////////////////////////// + +void FileSource::swap(FileSource& other) noexcept +{ + tl::swap(m_isConstructed, other.m_isConstructed); + tl::swap(m_arena, other.m_arena); + tl::swap(m_offset, other.m_offset); + tl::swap(m_size, other.m_size); + tl::swap(m_optLastError, other.m_optLastError); +} + +////////////////////////////////////////////////////////////////////////// + +size_t FileSource::read(tl::span data) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + const uint64_t sz = tl::min(static_cast(data.size()), getSize() - m_offset); + if (sz == 0) + return 0; + + auto it = fast_io::read(toFH(m_arena), data.begin(), data.begin() + sz); + const int64_t sizeRead = std::distance(data.begin(), it); + if (sizeRead < 0) + { + int err = errno; + m_optLastError = tl::make_error(ErrorCode::SystemError, "Failed to read from file: ({}): {}", err, + strerror(err)); + TL_FAIL("{}", m_optLastError); + return 0; + } + m_offset += static_cast(sizeRead); + + return tl::narrow(sizeRead); +} + +////////////////////////////////////////////////////////////////////////// + +void FileSource::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (m_offset != offset) + m_offset = fast_io::seek(toFH(m_arena), offset, fast_io::seekdir::beg); +} + +////////////////////////////////////////////////////////////////////////// + +void FileSource::seekRel(int64_t offset) +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t FileSource::tell() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t FileSource::getSize() const +{ + if (m_optLastError.has_value() || !isValid()) + { + TL_FAIL(); + return 0; + } + + if (m_size == uint64_t(-1)) + m_size = fast_io::status(toFH(m_arena)).size; + + return m_size; +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional FileSource::getLastError() const +{ + tl::optional error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +bool FileSource::isEOS() const +{ + if (!isValid()) + return true; + return m_offset >= getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t FileSource::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/FolderPack.cpp b/src/FolderPack.cpp new file mode 100644 index 0000000..f15fcc8 --- /dev/null +++ b/src/FolderPack.cpp @@ -0,0 +1,119 @@ +#include "StdAfx.h" +#include "fs/FolderPack.h" +#include "fs/NativeFilesystem.h" +#include "tl/ptr.h" + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +FolderPack::FolderPack(AbsPath location) + : m_filesystem(tl::lend(native)) + , m_location(std::move(location)) +{ + if (auto isFileResult = m_filesystem->isFile(m_location)) + TL_ASSERT(isFileResult.value() == false, "'{}' is a file", m_location); +} + +////////////////////////////////////////////////////////////////////////// + +FolderPack::FolderPack(tl::lent_ref filesystem, AbsPath location) + : m_filesystem(std::move(filesystem)) + , m_location(std::move(location)) +{ + if (auto isFileResult = m_filesystem->isFile(m_location)) + TL_ASSERT(isFileResult.value() == false, "'{}' is a file", m_location); +} + +////////////////////////////////////////////////////////////////////////// + +OpenSourceResult FolderPack::openSource(const AbsPath& path, SourceFlags flags) const +{ + return m_filesystem->openSource(convertToUnderlyingPath(path), flags); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSourceResult FolderPack::openStreamSource(const AbsPath& path, SourceFlags flags) const +{ + return m_filesystem->openStreamSource(convertToUnderlyingPath(path), flags); +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSourceResult FolderPack::openMapSource(const AbsPath& path, MapView mapView, SourceFlags flags) const +{ + return m_filesystem->openMapSource(convertToUnderlyingPath(path), mapView, flags); +} + +////////////////////////////////////////////////////////////////////////// + +IsFileResult FolderPack::isFile(const AbsPath& path) const +{ + return m_filesystem->isFile(convertToUnderlyingPath(path)); +} + +////////////////////////////////////////////////////////////////////////// + +IsFolderResult FolderPack::isFolder(const AbsPath& path) const +{ + return m_filesystem->isFolder(convertToUnderlyingPath(path)); +} + +////////////////////////////////////////////////////////////////////////// + +ExistsResult FolderPack::exists(const AbsPath& path) const +{ + return m_filesystem->exists(convertToUnderlyingPath(path)); +} + +////////////////////////////////////////////////////////////////////////// + +GetStatResult FolderPack::getStat(const AbsPath& path) const +{ + return m_filesystem->getStat(convertToUnderlyingPath(path)); +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator FolderPack::enumerate(const AbsPath& path) const +{ + const AbsPath localPath = convertToUnderlyingPath(path); + for (EnumerateEntry ee : m_filesystem->enumerate(localPath)) + co_yield std::move(ee); +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator FolderPack::enumerateRecursively(const AbsPath& path) const +{ + const AbsPath localPath = convertToUnderlyingPath(path); + for (EnumerateEntry ee : m_filesystem->enumerateRecursively(localPath)) + co_yield std::move(ee); +} + +////////////////////////////////////////////////////////////////////////// + +AbsPath FolderPack::convertToUnderlyingPath(const AbsPath& path) const +{ + AbsPath filePath; + filePath.reserve(m_location.size() + path.size()); + filePath = m_location; + for (const tl::string& element : path) + filePath.push_back(element); + + return filePath; +} + +////////////////////////////////////////////////////////////////////////// + +ConvertToNativePathResult FolderPack::convertToNativePath(const AbsPath& path) const +{ + return m_filesystem->convertToNativePath(convertToUnderlyingPath(path)); +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/IFilesystem.cpp b/src/IFilesystem.cpp new file mode 100644 index 0000000..778ad40 --- /dev/null +++ b/src/IFilesystem.cpp @@ -0,0 +1,8 @@ +#include "StdAfx.h" +#include "fs/IFilesystem.h" + +#include "tl/assert.h" +#include "fs/AbsPath.inl" + +////////////////////////////////////////////////////////////////////////// + diff --git a/src/MapSourceView.cpp b/src/MapSourceView.cpp new file mode 100644 index 0000000..18605f6 --- /dev/null +++ b/src/MapSourceView.cpp @@ -0,0 +1,88 @@ +#include "StdAfx.h" +#include "fs/MapSourceView.h" +#include "fs/MemorySource.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +MapSourceView::MapSourceView(tl::lent_ref srcSource, uint64_t offset, size_t size) + : m_memView(constructMemView(*srcSource, offset, size)) + , m_memViewSource(m_memView) +{ +} + +////////////////////////////////////////////////////////////////////////// + +tl::span MapSourceView::constructMemView(IMapSource& srcSource, uint64_t offset, size_t size) const +{ + srcSource.seekBeg(offset); + return srcSource.map(size); +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional MapSourceView::getLastError() const +{ + return m_memViewSource.getLastError(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t MapSourceView::read(tl::span data) +{ + return m_memViewSource.read(data); +} + +////////////////////////////////////////////////////////////////////////// + +void MapSourceView::seekBeg(uint64_t offset) +{ + return m_memViewSource.seekBeg(offset); +} + +////////////////////////////////////////////////////////////////////////// + +void MapSourceView::seekRel(int64_t offset) +{ + return m_memViewSource.seekRel(offset); +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t MapSourceView::tell() const +{ + return m_memViewSource.tell(); +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t MapSourceView::getSize() const +{ + return m_memViewSource.getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +tl::span MapSourceView::map(size_t size) +{ + return m_memViewSource.map(size); +} + +////////////////////////////////////////////////////////////////////////// + +bool MapSourceView::isEOS() const +{ + return m_memViewSource.isEOS(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t MapSourceView::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// +} + diff --git a/src/MemorySink.cpp b/src/MemorySink.cpp new file mode 100644 index 0000000..7bdb07b --- /dev/null +++ b/src/MemorySink.cpp @@ -0,0 +1,259 @@ +#include "StdAfx.h" +#include "fs/MemorySink.h" +#include "fs/MemorySource.h" +#include + +#include "tl/narrow_cast.h" + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ + +/////////////////////////////////////////////////////////////////////////////// + +MemorySink::MemorySink(size_t reserve) +{ + m_data.reserve(reserve); +} + +/////////////////////////////////////////////////////////////////////////////// + +MemorySink::MemorySink(tl::memory_buffer&& buffer) + : m_offset(buffer.size()) + , m_data(std::move(buffer)) +{ +} + +////////////////////////////////////////////////////////////////////////// + +void MemorySink::reserve(size_t capacity) +{ + m_data.reserve(capacity); +} + +////////////////////////////////////////////////////////////////////////// + +void MemorySink::shrinkToFit() +{ + m_data.shrink_to_fit(); +} + +/////////////////////////////////////////////////////////////////////////////// + +size_t MemorySink::write(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + if (data.empty()) + return 0; + + const size_t totalSize = m_offset + data.size(); + if (m_data.size() < totalSize) + m_data.resize_uninitialized(totalSize); + + memcpy(m_data.data() + m_offset, data.data(), data.size()); + m_offset += data.size(); + return data.size(); +} + +/////////////////////////////////////////////////////////////////////////////// + +void MemorySink::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + size_t realOffset; + if constexpr (sizeof(uint64_t) > sizeof(size_t)) + realOffset = tl::narrow(offset); + else + realOffset = offset; + + if (m_offset != realOffset) + { + m_offset = realOffset; + if (m_data.size() < m_offset) + m_data.resize_uninitialized(m_offset); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void MemorySink::seekRel(int64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(newOffset)); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +uint64_t MemorySink::tell() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +/////////////////////////////////////////////////////////////////////////////// + +uint64_t MemorySink::getSize() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_data.size(); +} + +/////////////////////////////////////////////////////////////////////////////// + +tl::span MemorySink::getDataView() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + return tl::span{m_data.data(), m_data.size()}; +} + +/////////////////////////////////////////////////////////////////////////////// + +tl::memory_buffer MemorySink::getDataCopy() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + return m_data; +} + +/////////////////////////////////////////////////////////////////////////////// + +tl::memory_buffer MemorySink::getDataMove() +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + return std::move(m_data); +} + +/////////////////////////////////////////////////////////////////////////////// + +void MemorySink::clear() +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + m_offset = 0; + m_data.clear(); +} + +/////////////////////////////////////////////////////////////////////////////// + +void MemorySink::swap(MemorySource& source) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + tl::swap(source.m_data, m_data); + source.m_offset = 0; + m_offset = 0; +} + +/////////////////////////////////////////////////////////////////////////////// + +tl::span MemorySink::map(size_t size) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + + const size_t remainingBytes = m_data.size() - m_offset; + const size_t bytesToMap = tl::min(size, remainingBytes); + if (bytesToMap == 0) + return {}; + + const tl::span mView(&m_data[m_offset], bytesToMap); + m_offset += bytesToMap; + + TL_ASSERT(m_offset <= getSize()); + + return mView; +} + +/////////////////////////////////////////////////////////////////////////////// + +tl::optional MemorySink::getLastError() const +{ + auto error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +size_t MemorySink::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +void MemorySink::preReserve(size_t size) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + m_data.reserve(m_data.size() + size); +} + +////////////////////////////////////////////////////////////////////////// + +tl::memory_buffer MemorySink::extractData(MemorySink sink) +{ + return sink.getDataMove(); +} + +/////////////////////////////////////////////////////////////////////////////// + +} + diff --git a/src/MemorySource.cpp b/src/MemorySource.cpp new file mode 100644 index 0000000..fbd5180 --- /dev/null +++ b/src/MemorySource.cpp @@ -0,0 +1,177 @@ +#include "StdAfx.h" +#include "fs/MemorySource.h" +#include "fs/MemorySink.h" +#include "tl/narrow_cast.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +MemorySource::MemorySource(tl::span data) +{ + m_data.resize_uninitialized(data.size()); + if (!data.empty()) + memcpy(m_data.data(), data.data(), data.size()); +} + +////////////////////////////////////////////////////////////////////////// + +MemorySource::MemorySource(MemorySink& sink) +{ + sink.swap(*this); +} + +////////////////////////////////////////////////////////////////////////// + +MemorySource::MemorySource(const tl::memory_buffer& memoryBuffer) +{ + TL_ASSERT(!memoryBuffer.empty()); + m_data = memoryBuffer; +} + +////////////////////////////////////////////////////////////////////////// + +size_t MemorySource::read(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + const size_t sz = tl::min(data.size(), m_data.size() - m_offset); + if (sz == 0) + return 0; + + memcpy(data.data(), m_data.data() + m_offset, sz); + m_offset += sz; + TL_ASSERT(m_offset <= getSize()); + return sz; +} + +////////////////////////////////////////////////////////////////////////// + +void MemorySource::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + size_t realOffset; + if constexpr (sizeof(uint64_t) > sizeof(size_t)) + realOffset = tl::narrow(offset); + else + realOffset = offset; + + m_offset = tl::min(realOffset, m_data.size()); +} + +////////////////////////////////////////////////////////////////////////// + +void MemorySource::seekRel(int64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + if (offset) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t MemorySource::tell() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t MemorySource::getSize() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_data.size(); +} + +////////////////////////////////////////////////////////////////////////// + +tl::span MemorySource::map(size_t size) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + + const size_t remainingBytes = m_data.size() - m_offset; + const size_t bytesToMap = tl::min(size, remainingBytes); + + if (bytesToMap == 0) + return {}; + + const tl::span mView(&m_data[m_offset], bytesToMap); + m_offset += bytesToMap; + + TL_ASSERT(m_offset <= getSize()); + + return mView; +} + +////////////////////////////////////////////////////////////////////////// + +void MemorySource::swap(MemorySink& sink) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + sink.swap(*this); +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional MemorySource::getLastError() const +{ + auto error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +bool MemorySource::isEOS() const +{ + return m_offset >= getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t MemorySource::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/MemoryViewSource.cpp b/src/MemoryViewSource.cpp new file mode 100644 index 0000000..875f10d --- /dev/null +++ b/src/MemoryViewSource.cpp @@ -0,0 +1,146 @@ +#include "StdAfx.h" +#include "fs/MemoryViewSource.h" + +#include "tl/narrow_cast.h" + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ + +////////////////////////////////////////////////////////////////////////// + +MemoryViewSource::MemoryViewSource(tl::span memView) + : m_memView(memView) +{ + +} + +////////////////////////////////////////////////////////////////////////// + + +size_t MemoryViewSource::read(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + const size_t sz = tl::min(data.size(), m_memView.size() - tl::narrow(m_offset)); + if (sz == 0) + return 0; + + memcpy(data.data(), m_memView.data() + tl::narrow(m_offset), sz); + m_offset += sz; + TL_ASSERT(m_offset <= getSize()); + return sz; +} + +////////////////////////////////////////////////////////////////////////// + +void MemoryViewSource::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + m_offset = tl::min(offset, m_memView.size()); +} + +////////////////////////////////////////////////////////////////////////// + +void MemoryViewSource::seekRel(int64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t MemoryViewSource::tell() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t MemoryViewSource::getSize() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_memView.size(); +} + +////////////////////////////////////////////////////////////////////////// + +tl::span MemoryViewSource::map(size_t size) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + + const size_t remainingBytes = m_memView.size() - tl::narrow(m_offset); + const size_t bytesToMap = tl::min(size, remainingBytes); + if (bytesToMap == 0) + return {}; + + const tl::span mView(&m_memView[tl::narrow(m_offset)], bytesToMap); + m_offset += bytesToMap; + + TL_ASSERT(m_offset <= getSize()); + + return mView; +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional MemoryViewSource::getLastError() const +{ + auto error = m_optLastError; + m_optLastError = tl::nullopt; + return error; +} + +////////////////////////////////////////////////////////////////////////// + +bool MemoryViewSource::isEOS() const +{ + return m_offset >= getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t MemoryViewSource::getPreferredBufferSize() const +{ + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/NativeFilesystem.cpp b/src/NativeFilesystem.cpp new file mode 100644 index 0000000..773ef35 --- /dev/null +++ b/src/NativeFilesystem.cpp @@ -0,0 +1,663 @@ +#include "StdAfx.h" +#include "fs/NativeFilesystem.h" + +#include "fs/FileSource.h" +#include "fs/FileSink.h" +#include "fs/FileMapSource.h" +#include "fs/FileMapSink.h" +#include "fs/NativeUtils.h" +#include "fs/BufferedFileSource.h" +#include "tl/assert.h" +#include +#include "fs/CopyStream.h" + + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ +namespace native_utils +{ +#if defined(TL_PLATFORM_WINDOWS_DESKTOP_APP) +int isCapitalizationCorrect(const tl::string& str, HANDLE handle); +#endif +} + +////////////////////////////////////////////////////////////////////////// +FS_API NativeFilesystem native(NativeFilesystem::CheckFlag::ValidatePathCase); +FS_API NativeFilesystem nativeRaw; +////////////////////////////////////////////////////////////////////////// + +std::filesystem::path toStdPath(const AbsPath& path) noexcept +{ + const tl::string str = path.str(); + return { str.data(), str.data() + str.size() }; +} +AbsPath toFSAbsPath(const std::filesystem::path& path) noexcept +{ + if (!path.is_absolute()) + return {}; + auto str = path.u8string(); + return {(char*)str.data(), str.size()}; +} +void toFSAbsPath(AbsPath& dst, const std::filesystem::path& path) noexcept +{ + if (!path.is_absolute()) + return; + auto str = path.u8string(); + dst.parse((char*)str.data(), str.size()); +} +RelPath toFSRelPath(const std::filesystem::path& path) noexcept +{ + if (!path.is_relative()) + return {}; + auto str = path.u8string(); + return {(char*)str.data(), str.size()}; +} + +//fix for the fact that the time_time clock doesn't have a from/to_time_t on some compilers +std::time_t to_time_t(std::filesystem::file_time_type tp) +{ + using namespace std::chrono; +#ifdef TL_PLATFORM_WINDOWS_FAMILY + return system_clock::to_time_t(utc_clock::to_sys(file_clock::to_utc(tp))); +#else + return system_clock::to_time_t(file_clock::to_sys(tp)); +#endif +} +std::filesystem::file_time_type from_time_t(std::time_t time) +{ + using namespace std::chrono; +#ifdef TL_PLATFORM_WINDOWS_FAMILY + return file_clock::from_utc(utc_clock::from_sys(system_clock::from_time_t(time))); +#else + return file_clock::from_sys(system_clock::from_time_t(time)); +#endif +} + +////////////////////////////////////////////////////////////////////////// + +#if defined(TL_PLATFORM_WIN_API) +//Returns the last Win32 error, in string format. Returns an empty string if there is no error. +std::string getErrorAsString(int error) +{ + LPSTR messageBuffer = nullptr; + const size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&messageBuffer, 0, nullptr); + + std::string message(messageBuffer, size); + + //Free the buffer. + LocalFree(messageBuffer); + + return message; +} + +////////////////////////////////////////////////////////////////////////// + +std::string getErrorAsString() +{ + return getErrorAsString(GetLastError()); +} + +////////////////////////////////////////////////////////////////////////// + +#elif defined(TL_PLATFORM_POSIX_API) +//Returns the last Win32 error, in string format. Returns an empty string if there is no error. +std::string getErrorAsString(int error) +{ + return std::string(strerror(error)); +} +std::string getErrorAsString() +{ + return getErrorAsString(errno); +} + +#endif + +////////////////////////////////////////////////////////////////////////// + +NativeFilesystem::NativeFilesystem(CheckFlags checkFlags) + : m_checkFlags(checkFlags) +{ +} + +////////////////////////////////////////////////////////////////////////// + +OpenSourceResult NativeFilesystem::openSource(const AbsPath& path, SourceFlags flags) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + if (flags.test(SourceFlag::Unbuffered)) + { + tl::unique_ref source = tl::make_unique(path, flags); + if (source->isValid()) + return std::move(source); + + tl::optional sourceErrorOpt = source->getLastError(); + if (sourceErrorOpt.has_value()) + return *sourceErrorOpt; + } + else + { + FileSource source(path, flags); + if (source.isValid()) + return tl::make_unique(std::move(source), 32768); + + tl::optional sourceErrorOpt = source.getLastError(); + if (sourceErrorOpt.has_value()) + return *sourceErrorOpt; + } + + return tl::make_error(ErrorCode::SystemError, "Cannot open file '{}': {}", path, getErrorAsString()); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSourceResult NativeFilesystem::openStreamSource(const AbsPath& path, SourceFlags flags) const +{ + OUTCOME_TRY(auto source, openSource(path, flags)); + return source; +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSourceResult NativeFilesystem::openMapSource(const AbsPath& path, MapView mapView, SourceFlags flags) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + tl::unique_ref source(new FileMapSource(path, mapView.offset, mapView.size)); + if (source->isValid()) + return std::move(source); + + tl::optional sourceErrorOpt = source->getLastError(); + if (sourceErrorOpt.has_value()) + return *sourceErrorOpt; + + return tl::make_error(ErrorCode::SystemError, "Cannot open file '{}': {}", path, getErrorAsString()); +} + +////////////////////////////////////////////////////////////////////////// + +OpenSinkResult NativeFilesystem::openSink(const AbsPath& path, Mode mode, SinkFlags flags) +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + tl::unique_ref source(new FileSink(path, mode, flags)); + if (source->isValid()) + return std::move(source); + + tl::optional sourceErrorOpt = source->getLastError(); + if (sourceErrorOpt.has_value()) + return *sourceErrorOpt; + + return tl::make_error(ErrorCode::SystemError, "Cannot open file '{}': {}", path, getErrorAsString()); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSinkResult NativeFilesystem::openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags) +{ + OUTCOME_TRY(auto source, openSink(path, mode, flags)); + return source; +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSinkResult NativeFilesystem::openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags) +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + tl::unique_ref source(new FileMapSink(path, size, mode, flags)); + if (source->isValid()) + return std::move(source); + + tl::optional sourceErrorOpt = source->getLastError(); + if (sourceErrorOpt.has_value()) + return *sourceErrorOpt; + + return tl::make_error(ErrorCode::SystemError, "Cannot open file '{}': {}", path, getErrorAsString()); +} + +////////////////////////////////////////////////////////////////////////// + +ConvertToNativePathResult NativeFilesystem::convertToNativePath(const AbsPath& path) const +{ + return path; +} + +////////////////////////////////////////////////////////////////////////// + +ExistsResult NativeFilesystem::exists(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + const bool v = std::filesystem::exists(toStdPath(path), ec); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return v; +} + +////////////////////////////////////////////////////////////////////////// + +IsFileResult NativeFilesystem::isFile(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + const bool v = std::filesystem::is_regular_file(toStdPath(path), ec); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec == std::errc::no_such_file_or_directory) + return false; + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return v; +} + +////////////////////////////////////////////////////////////////////////// + +IsFolderResult NativeFilesystem::isFolder(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFolder(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + const bool v = std::filesystem::is_directory(toStdPath(path), ec); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec == std::errc::no_such_file_or_directory) + return false; + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return v; +} + +////////////////////////////////////////////////////////////////////////// + +RenameResult NativeFilesystem::rename(const AbsPath& path, const AbsPath& newPath) +{ + if (!path.is_valid() || !newPath.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + std::filesystem::rename(toStdPath(path), toStdPath(newPath), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +CopyResult NativeFilesystem::copy(const AbsPath& path, const AbsPath& newPath) +{ + if (!path.is_valid() || !newPath.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + const bool v = std::filesystem::copy_file(toStdPath(path), toStdPath(newPath), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + if (!v) + return tl::make_error(ErrorCode::SystemError, "Unknown error"); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +MakeHardLinkResult NativeFilesystem::makeHardLink(const AbsPath& sourcePath, const AbsPath& linkPath) +{ + if (!sourcePath.is_valid() || !linkPath.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && + (native_utils::validateCaseSensitiveFilename(sourcePath) == 0 || native_utils::validateCaseSensitiveFilename(linkPath) == 0)) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + std::filesystem::create_hard_link(toStdPath(sourcePath), toStdPath(linkPath), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +MakeSoftLinkResult NativeFilesystem::makeSymLink(const AbsPath& sourcePath, const AbsPath& linkPath) +{ + if (!sourcePath.is_valid() || !linkPath.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && + (native_utils::validateCaseSensitiveFilename(sourcePath) == 0 || native_utils::validateCaseSensitiveFilename(linkPath) == 0)) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + OUTCOME_TRY(const bool isFolder, isFolder(sourcePath)); + + std::error_code ec; + if (isFolder) + std::filesystem::create_directory_symlink(toStdPath(sourcePath), toStdPath(linkPath), ec); + else + std::filesystem::create_symlink(toStdPath(sourcePath), toStdPath(linkPath), ec); + + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +MakeFolderResult NativeFilesystem::makeFolder(const AbsPath& path) +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + OUTCOME_TRY(const bool isFolder, isFolder(path)); + if (isFolder) + return tl::success(); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFolder(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + std::filesystem::create_directories(toStdPath(path), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +RemoveResult NativeFilesystem::remove(const AbsPath& path) +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + std::filesystem::remove(toStdPath(path), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +RemoveRecursivelyResult NativeFilesystem::removeRecursively(const AbsPath& path) +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFolder(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + OUTCOME_TRY(const bool isFolder, isFolder(path)); + + std::error_code ec; + if (isFolder) + std::filesystem::remove_all(toStdPath(path), ec); + else + std::filesystem::remove(toStdPath(path), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +SetWriteTimeResult NativeFilesystem::setWriteTime(const AbsPath& path, time_t time) +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + tl::string pathStr = path.str(); + + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + std::filesystem::last_write_time(toStdPath(path), from_time_t(time), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +GetStatResult NativeFilesystem::getStat(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFilename(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + struct stat s; + if (const int res = stat(path.str().c_str(), &s); res != 0) + { + if (res == ENOENT || res == ENOTDIR) + return tl::make_error(ErrorCode::NotFound, "Not found"); + if (res == EACCES) + return tl::make_error(ErrorCode::NotAllowed, "Access denied"); + return tl::make_error(ErrorCode::SystemError, "System error"); + } + + return Stat{ static_cast(s.st_size), s.st_mtime, s.st_atime }; +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator NativeFilesystem::enumerate(const AbsPath& path) const +{ + TL_ASSERT(path.is_valid()); + + IsFolderResult isFolderResult = isFolder(path); + if (isFolderResult.has_error() || isFolderResult.value() == false) + co_return; + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFolder(path) == 0) + { + TL_FAIL("Bad path case!"); + co_return; + } + + for (auto const& entry : std::filesystem::directory_iterator{ toStdPath(path) }) + { + const auto status = entry.status(); + const auto type = status.type(); + if (type == std::filesystem::file_type::directory) + { + auto str = entry.path().filename().u8string(); + co_yield { RelPath((char*)str.data(), str.size()), true }; + } + else if (type == std::filesystem::file_type::regular) + { + auto str = entry.path().filename().u8string(); + co_yield { RelPath((char*)str.data(), str.size()), false }; + } + } +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator NativeFilesystem::enumerateRecursively(const AbsPath& path) const +{ + TL_ASSERT(path.is_valid()); + + IsFolderResult isFolderResult = isFolder(path); + if (isFolderResult.has_error() || isFolderResult.value() == false) + co_return; + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFolder(path) == 0) + { + TL_FAIL("Bad path case!"); + co_return; + } + + AbsPath eePath; + const auto stdRootPath = toStdPath(path); + for (auto const& entry : std::filesystem::recursive_directory_iterator{ stdRootPath }) + { + const auto status = entry.status(); + const auto type = status.type(); + if (type == std::filesystem::file_type::directory) + { + toFSAbsPath(eePath, entry.path()); + co_yield{ path.path_to(eePath), true }; + } + else if (type == std::filesystem::file_type::regular) + { + toFSAbsPath(eePath, entry.path()); + co_yield{ path.path_to(eePath), false }; + } + } +} + +////////////////////////////////////////////////////////////////////////// + +AbsPath NativeFilesystem::getCurrentFolder() const +{ + std::error_code ec; + const auto path = std::filesystem::current_path(ec); + return toFSAbsPath(path); +} + +////////////////////////////////////////////////////////////////////////// + +tl::result NativeFilesystem::setCurrentFolder(const AbsPath& path) +{ + if (!path.is_valid()) + return tl::make_error(Error::code_t::BadPath, "Path empty"); + + if (m_checkFlags.test(CheckFlag::ValidatePathCase) && native_utils::validateCaseSensitiveFolder(path) == 0) + return tl::make_error(ErrorCode::BadPath, "Bad path case!"); + + std::error_code ec; + std::filesystem::current_path(toStdPath(path), ec); + if (ec == std::errc::no_such_file_or_directory) + return tl::make_error(ErrorCode::NotFound, "Not found: {}", ec.message()); + if (ec == std::errc::permission_denied) + return tl::make_error(ErrorCode::NotAllowed, "Access denied: {}", ec.message()); + if (ec) + return tl::make_error(ErrorCode::SystemError, "System error: {}", ec.message()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +NativeFilesystem::EnumerateFilesResult NativeFilesystem::enumerateFiles(const AbsPath& fullPath) const +{ + tl::vector files; + for (EnumerateEntry ee: enumerate(fullPath)) + { + if (!ee.isFolder) + { + files.reserve(1024); + files.push_back(ee.path.back()); + } + } + return files; +} + +////////////////////////////////////////////////////////////////////////// + +NativeFilesystem::EnumerateFoldersResult NativeFilesystem::enumerateFolders(const AbsPath& fullPath) const +{ + tl::vector folders; + for (EnumerateEntry ee : enumerate(fullPath)) + { + if (ee.isFolder) + { + folders.reserve(1024); + folders.push_back(ee.path.back()); + } + } + return folders; +} + +////////////////////////////////////////////////////////////////////////// + +AbsPath NativeFilesystem::makeAbsPath(const tl::string& string) const +{ + return AbsPath::from_string(string, getCurrentFolder()); +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/NativeUtils.cpp b/src/NativeUtils.cpp new file mode 100644 index 0000000..e29b5eb --- /dev/null +++ b/src/NativeUtils.cpp @@ -0,0 +1,213 @@ +#include "StdAfx.h" +#include "fs/NativeUtils.h" +#include "fs/AbsPath.h" +#include "tl/finally.h" +#include "tl/fixed_string.h" + +#if defined(TL_PLATFORM_WINDOWS_FAMILY) +# include +# include +# include +# include //_utime + +#elif defined(TL_PLATFORM_POSIX_API) +# include +# include //utime +# include +# include +#endif + +#include +#include "tl/narrow_cast.h" + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +namespace native_utils +{ + +////////////////////////////////////////////////////////////////////////// + +#if defined(TL_PLATFORM_WINDOWS_DESKTOP_APP) + +////////////////////////////////////////////////////////////////////////// + +static bool isCapitalizationCorrect(const WCHAR* name, HANDLE handle) noexcept +{ + tl::fixed_string buffer; + buffer.resize(buffer.max_size()); + DWORD length = GetFinalPathNameByHandleW(handle, buffer.data(), static_cast(buffer.size()), FILE_NAME_NORMALIZED | VOLUME_NAME_NONE); + if (length > buffer.size()) + { + buffer.resize(length); + length = GetFinalPathNameByHandleW(handle, buffer.data(), static_cast(buffer.size()), FILE_NAME_NORMALIZED | VOLUME_NAME_NONE); + TL_ASSERT(length + 1== buffer.size()); + } + + return wcscmp(buffer.data(), name + 2) == 0; +} + +////////////////////////////////////////////////////////////////////////// + +bool isCapitalizationCorrect(const tl::string& str, HANDLE handle) noexcept +{ + const size_t wNameSize = str.size() + 1; + tl::fixed_vector wName(wNameSize); + swprintf(wName.data(), wNameSize, L"%hs", str.c_str()); + return isCapitalizationCorrect(wName.data(), handle); +} + +////////////////////////////////////////////////////////////////////////// + +bool isCapitalizationCorrect(const AbsPath& path, HANDLE handle) noexcept +{ + return isCapitalizationCorrect(path.str(), handle); +} + +////////////////////////////////////////////////////////////////////////// + +static bool isFileCapitalizationCorrect(const AbsPath& filePath) noexcept +{ + const tl::string fileStr = filePath.str(); + const char* fileName = fileStr.c_str(); + + const size_t wNameSize = fileStr.size() + 1; + tl::fixed_vector wName(wNameSize); + swprintf(wName.data(), wNameSize, L"%hs", fileName); + + const HANDLE hFile = CreateFile2(wName.data(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, OPEN_EXISTING, nullptr); + if (hFile == INVALID_HANDLE_VALUE) + return true; //couldn't find it, so it must be a new file + + const bool result = isCapitalizationCorrect(wName.data(), hFile); + CloseHandle(hFile); + + return result; +} + +////////////////////////////////////////////////////////////////////////// + +static bool isFolderCapitalizationCorrect(const AbsPath& folderPath) noexcept +{ + const tl::string folderStr = folderPath.str(); + const char* folderName = folderStr.c_str(); + + const size_t wNameSize = folderStr.size() + 1; + tl::fixed_vector wName(wNameSize); + swprintf(wName.data(), wNameSize, L"%hs", folderName); + + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters = {}; + extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); + extendedParameters.dwFileFlags = FILE_FLAG_BACKUP_SEMANTICS; + const HANDLE hFolder = CreateFile2(wName.data(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, OPEN_EXISTING, &extendedParameters); + if (hFolder == INVALID_HANDLE_VALUE) + return true; //couldn't find it, so it must be a new folder + + const bool result = isCapitalizationCorrect(wName.data(), hFolder); + CloseHandle(hFolder); + + return result; +} + +#endif //TL_PLATFORM_WINDOWS_DESKTOP_APP + +////////////////////////////////////////////////////////////////////////// + +bool validateCaseSensitiveFilename(const AbsPath& filepath) noexcept +{ +#ifdef TL_PLATFORM_WINDOWS_DESKTOP_APP + return isFileCapitalizationCorrect(filepath); +#else + return true; +#endif +} + +////////////////////////////////////////////////////////////////////////// + +bool validateCaseSensitiveFilename(const AbsPath& filepath, int fd) noexcept +{ +#ifdef TL_PLATFORM_WINDOWS_DESKTOP_APP + const HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) + return false; + + return isCapitalizationCorrect(filepath, handle); +#else + return true; +#endif +} + +////////////////////////////////////////////////////////////////////////// + +#ifdef TL_PLATFORM_WINDOWS_FAMILY +bool validateCaseSensitiveFilename(const AbsPath& filepath, HANDLE handle) noexcept +{ +#ifdef TL_PLATFORM_WINDOWS_DESKTOP_APP + if (handle == INVALID_HANDLE_VALUE) + return false; + + return isCapitalizationCorrect(filepath, handle); +#else + return true; +#endif +} +#endif +////////////////////////////////////////////////////////////////////////// + +bool validateCaseSensitiveFolder(const AbsPath& folderPath) noexcept +{ +#ifdef TL_PLATFORM_WINDOWS_DESKTOP_APP + return isFolderCapitalizationCorrect(folderPath); +#else + return true; +#endif +} + +//////////////////////////////////////////////////////////////////////// + +wstring_t utf8To16(tl::span str) noexcept +{ + TL_ASSERT(str.size() > 0); + + const size_t maxNameSize = str.size() + 1; + wstring_t ret; + ret.resize(maxNameSize); + +#ifdef TL_PLATFORM_WINDOWS_FAMILY + const int charactersCopied = MultiByteToWideChar(CP_UTF8, 0, str.data(), -1, const_cast(ret.data()), tl::narrow(maxNameSize)); +#else + const int charactersCopied = swprintf(const_cast(ret.data()), maxNameSize, L"%hs", utf8String); +#endif + + if (charactersCopied <= 0) + return {}; + + ret.resize(charactersCopied); + return ret; +} + +//////////////////////////////////////////////////////////////////////// + +string_t utf16To8(tl::span str) noexcept +{ + TL_ASSERT(str.size() > 0); + + string_t ret; + const size_t maxNameSize = str.size() + 1; + ret.resize(maxNameSize); + + const int charactersCopied = snprintf(const_cast(ret.data()), maxNameSize, "%ls", str.data()); + if (charactersCopied <= 0) + return {}; + + ret.resize(charactersCopied); + return ret; +} + +//////////////////////////////////////////////////////////////////////// + +} +} diff --git a/src/ProcessStream.cpp b/src/ProcessStream.cpp new file mode 100644 index 0000000..6af4115 --- /dev/null +++ b/src/ProcessStream.cpp @@ -0,0 +1,43 @@ +#include "StdAfx.h" +#include "fs/ProcessStream.h" +#include "fs/IStreamSource.h" +#include "tl/memory_buffer.h" +#include "tl/narrow_cast.h" + +namespace fs +{ +static constexpr size_t k_defaultBufferSize = 100 * 1024; + + +ProcessStreamResult processStream(IStreamSource& i_source, const ProcessDataCallback& i_dataCallback, size_t i_bufferSize) +{ + size_t bufferSize = i_bufferSize; + if (bufferSize == 0) + bufferSize = i_source.getPreferredBufferSize(); + + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + tl::memory_buffer buffer; + buffer.resize_uninitialized(bufferSize); + + do + { + const uint64_t read = i_source.read(buffer); + tl::optional optLastError = i_source.getLastError(); + if (optLastError.has_value()) + { + return *optLastError; + } + if (read == 0) + return tl::success(); + + if (i_dataCallback) + i_dataCallback({ buffer.data(), tl::narrow(read) }); + } while (!i_source.isEOS()); + + return tl::success(); +} + + +} diff --git a/src/ReadStream.cpp b/src/ReadStream.cpp new file mode 100644 index 0000000..2af7cee --- /dev/null +++ b/src/ReadStream.cpp @@ -0,0 +1,124 @@ +#include "StdAfx.h" +#include "fs/ReadStream.h" +#include "fs/IStreamSource.h" +#include "tl/narrow_cast.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +static constexpr size_t k_defaultBufferSize = 4u * 1024 * 1024; + +////////////////////////////////////////////////////////////////////////// + +ReadStreamResult readStream(tl::memory_buffer& buffer, IStreamSource& source, size_t bufferSize) +{ + if (bufferSize == 0) + bufferSize = source.getPreferredBufferSize(); + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + do + { + const size_t offset = buffer.size(); + buffer.resize_uninitialized(offset + bufferSize); + const uint64_t read = source.read({ buffer.data() + offset, bufferSize }); + tl::optional optLastError = source.getLastError(); + if (optLastError.has_value()) + return std::move(*optLastError); + + if (read < bufferSize) + { + buffer.resize_uninitialized(offset + tl::narrow(read)); + break; + } + } while (!source.isEOS()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +ReadStreamResult readStream(eastl::string& buffer, IStreamSource& source, size_t bufferSize) +{ + if (bufferSize == 0) + bufferSize = source.getPreferredBufferSize(); + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + do + { + const size_t offset = buffer.size(); + buffer.resize(offset + bufferSize); + const uint64_t read = source.read({ reinterpret_cast(buffer.data()) + offset, bufferSize }); + tl::optional optLastError = source.getLastError(); + if (optLastError.has_value()) + { + buffer.clear(); + return std::move(*optLastError); + } + + if (read < bufferSize) + { + buffer.resize(offset + tl::narrow(read)); + break; + } + } while (!source.isEOS()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +ReadStreamResult readStream(std::string& buffer, IStreamSource& source, size_t bufferSize) +{ + if (bufferSize == 0) + bufferSize = source.getPreferredBufferSize(); + if (bufferSize == 0) + bufferSize = k_defaultBufferSize; + + do + { + const size_t offset = buffer.size(); + buffer.resize(offset + bufferSize); + const uint64_t read = source.read({ reinterpret_cast(buffer.data()) + offset, bufferSize }); + tl::optional optLastError = source.getLastError(); + if (optLastError.has_value()) + return std::move(*optLastError); + + if (read < bufferSize) + { + buffer.resize(offset + tl::narrow(read)); + break; + } + } while (!source.isEOS()); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +ReadStreamResult readFile(tl::memory_buffer& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return readStream(buffer, *source, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// + +ReadStreamResult readFile(eastl::string& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return readStream(buffer, *source, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// + +ReadStreamResult readFile(std::string& buffer, IFilesystem& filesystem, const AbsPath& filePath, size_t bufferSize) +{ + OUTCOME_TRY(const auto source, filesystem.openStreamSource(filePath, IFilesystem::SourceFlags(IFilesystem::SourceFlag::SequentialHint, IFilesystem::SourceFlag::Unbuffered))); + return readStream(buffer, *source, bufferSize); +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/SourceView.cpp b/src/SourceView.cpp new file mode 100644 index 0000000..9eb1b70 --- /dev/null +++ b/src/SourceView.cpp @@ -0,0 +1,125 @@ +#include "StdAfx.h" +#include "fs/SourceView.h" + +#include "tl/narrow_cast.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +SourceView::SourceView(tl::unique_ref&& srcSource, uint64_t offset, uint64_t size) + : m_srcSource(std::move(srcSource)) + , m_viewOffset(offset) + , m_viewSize(size) +{ + m_srcSource->seekBeg(offset); + m_optLastError = m_srcSource->getLastError(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t SourceView::read(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + uint64_t sz = tl::min(static_cast(data.size()), getSize() - m_offset); + if (sz == 0) + return 0; + + sz = m_srcSource->read({ data.data(), tl::narrow(sz) }); + m_optLastError = m_srcSource->getLastError(); + + m_offset += sz; + TL_ASSERT(m_offset <= getSize()); + return tl::narrow(sz); +} + +////////////////////////////////////////////////////////////////////////// + +void SourceView::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + m_offset = tl::min(offset, static_cast(m_viewSize)); + m_srcSource->seekBeg(m_viewOffset + m_offset); + m_optLastError = m_srcSource->getLastError(); +} + +////////////////////////////////////////////////////////////////////////// + +void SourceView::seekRel(int64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + if (offset != 0) + { + int64_t newOffset = static_cast(tell()); + newOffset += offset; + newOffset = tl::max(newOffset, 0); + seekBeg(static_cast(tl::min(newOffset, (int64_t)getSize()))); + } +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t SourceView::tell() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t SourceView::getSize() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_viewSize; +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional SourceView::getLastError() const +{ + return m_optLastError; +} + +////////////////////////////////////////////////////////////////////////// + +bool SourceView::isEOS() const +{ + return m_offset >= getSize(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t SourceView::getPreferredBufferSize() const +{ + return m_srcSource->getPreferredBufferSize(); +} + +////////////////////////////////////////////////////////////////////////// +} + + diff --git a/src/StdAfx.cpp b/src/StdAfx.cpp new file mode 100644 index 0000000..09f15b6 --- /dev/null +++ b/src/StdAfx.cpp @@ -0,0 +1 @@ +#include "StdAfx.h" \ No newline at end of file diff --git a/src/StdAfx.h b/src/StdAfx.h new file mode 100644 index 0000000..8dbdc63 --- /dev/null +++ b/src/StdAfx.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tl/result.h" +#include "tl/string.h" +#include "tl/format.h" +#include "tl/functional.h" +#include "tl/abs_path.h" + diff --git a/src/StreamSourceToSourceAdapter.cpp b/src/StreamSourceToSourceAdapter.cpp new file mode 100644 index 0000000..78592a7 --- /dev/null +++ b/src/StreamSourceToSourceAdapter.cpp @@ -0,0 +1,222 @@ +#include "StdAfx.h" + +#include "fs/StreamSourceToSourceAdapter.h" + +#include + +#include "tl/narrow_cast.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +static constexpr size_t k_chunkSize = 10 * 1024; + +StreamSourceToSourceAdapter::StreamSourceToSourceAdapter(tl::unique_ref&& srcSource) + : m_srcSource(std::move(srcSource)) +{ +} + +////////////////////////////////////////////////////////////////////////// + +tl::optional StreamSourceToSourceAdapter::getLastError() const +{ + return m_optLastError; +} + +////////////////////////////////////////////////////////////////////////// + +size_t StreamSourceToSourceAdapter::read(tl::span data) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + if (data.empty()) + return 0; + + if (!m_readAll) + { + // do we have enough data in the buffer? + if (m_offset + data.size() > m_data.size()) // nope! read some more + { + const size_t toRead = tl::narrow(m_offset + data.size() - m_data.size()); + + const size_t dataSize = m_data.size(); + m_data.resize_uninitialized(dataSize + toRead); + const uint64_t read = m_srcSource->read({ m_data.data() + dataSize, toRead }); + if (read < toRead || m_srcSource->isEOS()) + { + m_data.resize_uninitialized(dataSize + tl::narrow(read)); + m_data.shrink_to_fit(); + m_readAll = true; + } + m_optLastError = m_srcSource->getLastError(); + } + } + + TL_ASSERT(m_data.size() >= m_offset); + + const size_t sz = tl::min(data.size(), m_data.size() - tl::narrow(m_offset)); + if (sz > 0) + memcpy(data.data(), m_data.data() + m_offset, sz); + + m_offset += sz; + return sz; +} + +////////////////////////////////////////////////////////////////////////// + +void StreamSourceToSourceAdapter::seekBeg(uint64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + if (offset < m_data.size()) + m_offset = offset; + else + { + readAll(); + m_offset = tl::min(offset, static_cast(m_data.size())); + } +} + +////////////////////////////////////////////////////////////////////////// + +void StreamSourceToSourceAdapter::seekRel(int64_t offset) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return; + } + + uint64_t newOffset = m_offset; + if (offset > 0) + newOffset += (uint64_t)offset; + else + newOffset -= (uint64_t)(-offset); + + seekBeg(newOffset); +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t StreamSourceToSourceAdapter::tell() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + return m_offset; +} + +////////////////////////////////////////////////////////////////////////// + +uint64_t StreamSourceToSourceAdapter::getSize() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return 0; + } + + readAll(); + if (m_optLastError != tl::nullopt) + return 0; + + return m_data.size(); +} + +////////////////////////////////////////////////////////////////////////// + +bool StreamSourceToSourceAdapter::isEOS() const +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return true; + } + + // we're not even at the end of our data.size, so definitely not EOS + if (m_offset < m_data.size()) + return false; + + // if we finished the local buffer, check the src source + TL_ASSERT(m_offset == m_data.size()); + return m_srcSource->isEOS(); +} + +////////////////////////////////////////////////////////////////////////// + +size_t StreamSourceToSourceAdapter::getPreferredBufferSize() const +{ + return m_srcSource->getPreferredBufferSize(); +} + +////////////////////////////////////////////////////////////////////////// + +tl::span StreamSourceToSourceAdapter::map(size_t size) +{ + if (m_optLastError.has_value()) + { + TL_FAIL(); + return {}; + } + + if (!m_readAll) + readAll(); + + if (m_optLastError.has_value()) + return {}; + + TL_ASSERT(m_data.size() >= m_offset); + + const uint64_t offset = m_offset; + const size_t maxSize = tl::min(m_data.size() - tl::narrow(m_offset), size); + + m_offset += maxSize; + + return {m_data.data() + offset, maxSize}; +} + +////////////////////////////////////////////////////////////////////////// + +void StreamSourceToSourceAdapter::readAll() const +{ + if (m_readAll) + return; + + const size_t chunkSize = m_srcSource->getPreferredBufferSize() > 0 ? m_srcSource->getPreferredBufferSize() : k_chunkSize; + + do + { + const size_t dataSize = m_data.size(); + m_data.resize_uninitialized(dataSize + chunkSize); + const uint64_t read = m_srcSource->read({ m_data.data() + dataSize, chunkSize }); + if (read < chunkSize || m_srcSource->isEOS()) + { + m_data.resize_uninitialized(dataSize + tl::narrow(read)); + break; + } + m_optLastError = m_srcSource->getLastError(); + if (m_optLastError != tl::nullopt) + { + m_data.clear(); + break; + } + } while (true); + + m_data.shrink_to_fit(); + m_readAll = true; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/WritableFolderPack.cpp b/src/WritableFolderPack.cpp new file mode 100644 index 0000000..15a7a0a --- /dev/null +++ b/src/WritableFolderPack.cpp @@ -0,0 +1,94 @@ +#include "StdAfx.h" +#include "fs/WritableFolderPack.h" +#include "fs/NativeFilesystem.h" +#include "Logger.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +WritableFolderPack::WritableFolderPack(AbsPath location) + : FolderPack(std::move(location)) +{ + MakeFolderResult result = fs::native.makeFolder(m_location); + if (result.has_error()) + LOGW("Could not create WritablePackFolder {} due to error {} ", m_location, result.error()); +} + +////////////////////////////////////////////////////////////////////////// + +WritableFolderPack::WritableFolderPack(tl::lent_ref filesystem, AbsPath location) + : FolderPack(std::move(filesystem), std::move(location)) +{ + MakeFolderResult result = m_filesystem->makeFolder(m_location); + if (result.has_error()) + LOGW("Could not create WritablePackFolder {} due to error {} ", m_location, result.error()); +} + +////////////////////////////////////////////////////////////////////////// + +OpenSinkResult WritableFolderPack::openSink(const AbsPath& path, Mode mode, SinkFlags flags) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->openSink(npath, mode, flags); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSinkResult WritableFolderPack::openStreamSink(const AbsPath& path, Mode mode, SinkFlags flags) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->openStreamSink(npath, mode, flags); +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSinkResult WritableFolderPack::openMapSink(const AbsPath& path, Mode mode, size_t size, SinkFlags flags) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->openMapSink(npath, mode, size, flags); +} + +////////////////////////////////////////////////////////////////////////// + +RemoveResult WritableFolderPack::remove(const AbsPath& path) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->remove(npath); +} + +////////////////////////////////////////////////////////////////////////// + +RenameResult WritableFolderPack::rename(const AbsPath& path, const AbsPath& newPath) +{ + OUTCOME_TRY(const auto filePath, convertToNativePath(path)); + OUTCOME_TRY(const auto newFilePath, convertToNativePath(newPath)); + return m_filesystem->rename(filePath, newFilePath); +} + +////////////////////////////////////////////////////////////////////////// + +MakeFolderResult WritableFolderPack::makeFolder(const AbsPath& path) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->makeFolder(npath); +} + +////////////////////////////////////////////////////////////////////////// + +RemoveRecursivelyResult WritableFolderPack::removeRecursively(const AbsPath& path) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->removeRecursively(npath); +} + +////////////////////////////////////////////////////////////////////////// + +SetWriteTimeResult WritableFolderPack::setWriteTime(const AbsPath& path, time_t time) +{ + OUTCOME_TRY(const auto npath, convertToNativePath(path)); + return m_filesystem->setWriteTime(npath, time); +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/mio/README.md b/src/mio/README.md new file mode 100644 index 0000000..a9ab409 --- /dev/null +++ b/src/mio/README.md @@ -0,0 +1,339 @@ +# mio +An easy to use header-only cross-platform C++11 memory mapping library with an MIT license. + +mio has been created with the goal to be easily includable (i.e. no dependencies) in any C++ project that needs memory mapped file IO without the need to pull in Boost. + +Please feel free to open an issue, I'll try to address any concerns as best I can. + +### Why? +Because memory mapping is the best thing since sliced bread! + +More seriously, the primary motivation for writing this library instead of using Boost.Iostreams, was the lack of support for establishing a memory mapping with an already open file handle/descriptor. This is possible with mio. + +Furthermore, Boost.Iostreams' solution requires that the user pick offsets exactly at page boundaries, which is cumbersome and error prone. mio, on the other hand, manages this internally, accepting any offset and finding the nearest page boundary. + +Albeit a minor nitpick, Boost.Iostreams implements memory mapped file IO with a `std::shared_ptr` to provide shared semantics, even if not needed, and the overhead of the heap allocation may be unnecessary and/or unwanted. +In mio, there are two classes to cover the two use-cases: one that is move-only (basically a zero-cost abstraction over the system specific mmapping functions), and the other that acts just like its Boost.Iostreams counterpart, with shared semantics. + +### How to create a mapping +NOTE: the file must exist before creating a mapping. + +There are three ways to map a file into memory: + +- Using the constructor, which throws a `std::system_error` on failure: +```c++ +mio::mmap_source mmap(path, offset, size_to_map); +``` +or you can omit the `offset` and `size_to_map` arguments, in which case the +entire file is mapped: +```c++ +mio::mmap_source mmap(path); +``` + +- Using the factory function: +```c++ +std::error_code error; +mio::mmap_source mmap = mio::make_mmap_source(path, offset, size_to_map, error); +``` +or: +```c++ +mio::mmap_source mmap = mio::make_mmap_source(path, error); +``` + +- Using the `map` member function: +```c++ +std::error_code error; +mio::mmap_source mmap; +mmap.map(path, offset, size_to_map, error); +``` +or: +```c++ +mmap.map(path, error); +``` +**NOTE:** The constructors **require** exceptions to be enabled. If you prefer +to build your projects with `-fno-exceptions`, you can still use the other ways. + +Moreover, in each case, you can provide either some string type for the file's path, or you can use an existing, valid file handle. +```c++ +#include +#include +#include +#include +// #include if using single header +#include + +int main() +{ + // NOTE: error handling omitted for brevity. + const int fd = open("file.txt", O_RDONLY); + mio::mmap_source mmap(fd, 0, mio::map_entire_file); + // ... +} +``` +However, mio does not check whether the provided file descriptor has the same access permissions as the desired mapping, so the mapping may fail. Such errors are reported via the `std::error_code` out parameter that is passed to the mapping function. + +**WINDOWS USERS**: This library *does* support the use of wide character types +for functions where character strings are expected (e.g. path parameters). + +### Example + +```c++ +#include +// #include if using single header +#include // for std::error_code +#include // for std::printf +#include +#include +#include + +int handle_error(const std::error_code& error); +void allocate_file(const std::string& path, const int size); + +int main() +{ + const auto path = "file.txt"; + + // NOTE: mio does *not* create the file for you if it doesn't exist! You + // must ensure that the file exists before establishing a mapping. It + // must also be non-empty. So for illustrative purposes the file is + // created now. + allocate_file(path, 155); + + // Read-write memory map the whole file by using `map_entire_file` where the + // length of the mapping is otherwise expected, with the factory method. + std::error_code error; + mio::mmap_sink rw_mmap = mio::make_mmap_sink( + path, 0, mio::map_entire_file, error); + if (error) { return handle_error(error); } + + // You can use any iterator based function. + std::fill(rw_mmap.begin(), rw_mmap.end(), 'a'); + + // Or manually iterate through the mapped region just as if it were any other + // container, and change each byte's value (since this is a read-write mapping). + for (auto& b : rw_mmap) { + b += 10; + } + + // Or just change one value with the subscript operator. + const int answer_index = rw_mmap.size() / 2; + rw_mmap[answer_index] = 42; + + // Don't forget to flush changes to disk before unmapping. However, if + // `rw_mmap` were to go out of scope at this point, the destructor would also + // automatically invoke `sync` before `unmap`. + rw_mmap.sync(error); + if (error) { return handle_error(error); } + + // We can then remove the mapping, after which rw_mmap will be in a default + // constructed state, i.e. this and the above call to `sync` have the same + // effect as if the destructor had been invoked. + rw_mmap.unmap(); + + // Now create the same mapping, but in read-only mode. Note that calling the + // overload without the offset and file length parameters maps the entire + // file. + mio::mmap_source ro_mmap; + ro_mmap.map(path, error); + if (error) { return handle_error(error); } + + const int the_answer_to_everything = ro_mmap[answer_index]; + assert(the_answer_to_everything == 42); +} + +int handle_error(const std::error_code& error) +{ + const auto& errmsg = error.message(); + std::printf("error mapping file: %s, exiting...\n", errmsg.c_str()); + return error.value(); +} + +void allocate_file(const std::string& path, const int size) +{ + std::ofstream file(path); + std::string s(size, '0'); + file << s; +} +``` + +`mio::basic_mmap` is move-only, but if multiple copies to the same mapping are needed, use `mio::basic_shared_mmap` which has `std::shared_ptr` semantics and has the same interface as `mio::basic_mmap`. +```c++ +#include + +mio::shared_mmap_source shared_mmap1("path", offset, size_to_map); +mio::shared_mmap_source shared_mmap2(std::move(mmap1)); // or use operator= +mio::shared_mmap_source shared_mmap3(std::make_shared(mmap1)); // or use operator= +mio::shared_mmap_source shared_mmap4; +shared_mmap4.map("path", offset, size_to_map, error); +``` + +It's possible to define the type of a byte (which has to be the same width as `char`), though aliases for the most common ones are provided by default: +```c++ +using mmap_source = basic_mmap_source; +using ummap_source = basic_mmap_source; + +using mmap_sink = basic_mmap_sink; +using ummap_sink = basic_mmap_sink; +``` +But it may be useful to define your own types, say when using the new `std::byte` type in C++17: +```c++ +using mmap_source = mio::basic_mmap_source; +using mmap_sink = mio::basic_mmap_sink; +``` + +Though generally not needed, since mio maps users requested offsets to page boundaries, you can query the underlying system's page allocation granularity by invoking `mio::page_size()`, which is located in `mio/page.hpp`. + +### Single Header File +Mio can be added to your project as a single header file simply by including `\single_include\mio\mio.hpp`. Single header files can be regenerated at any time by running the `amalgamate.py` script within `\third_party`. +``` +python amalgamate.py -c config.json -s ../include +``` + +## CMake +As a header-only library, mio has no compiled components. Nevertheless, a [CMake](https://cmake.org/overview/) build system is provided to allow easy testing, installation, and subproject composition on many platforms and operating systems. + +### Testing +Mio is distributed with a small suite of tests and examples. +When mio is configured as the highest level CMake project, this suite of executables is built by default. +Mio's test executables are integrated with the CMake test driver program, [CTest](https://cmake.org/cmake/help/latest/manual/ctest.1.html). + +CMake supports a number of backends for compilation and linking. + +To use a static configuration build tool, such as GNU Make or Ninja: + +```sh +cd +mkdir build +cd build + +# Configure the build +cmake -D CMAKE_BUILD_TYPE= \ + -G <"Unix Makefiles" | "Ninja"> .. + +# build the tests +< make | ninja | cmake --build . > + +# run the tests +< make test | ninja test | cmake --build . --target test | ctest > +``` + +To use a dynamic configuration build tool, such as Visual Studio or Xcode: + +```sh +cd +mkdir build +cd build + +# Configure the build +cmake -G <"Visual Studio 14 2015 Win64" | "Xcode"> .. + +# build the tests +cmake --build . --config + +# run the tests via ctest... +ctest --build-config + +# ... or via CMake build tool mode... +cmake --build . --config --target test +``` + +Of course the **build** and **test** steps can also be executed via the **all** and **test** targets, respectively, from within the IDE after opening the project file generated during the configuration step. + +Mio's testing is also configured to operate as a client to the [CDash](https://www.cdash.org/) software quality dashboard application. Please see the [Kitware documentation](https://cmake.org/cmake/help/latest/manual/ctest.1.html#dashboard-client) for more information on this mode of operation. + +### Installation + +Mio's build system provides an installation target and support for downstream consumption via CMake's [`find_package`](https://cmake.org/cmake/help/v3.0/command/find_package.html) intrinsic function. +CMake allows installation to an arbitrary location, which may be specified by defining `CMAKE_INSTALL_PREFIX` at configure time. +In the absense of a user specification, CMake will install mio to conventional location based on the platform operating system. + +To use a static configuration build tool, such as GNU Make or Ninja: + +```sh +cd +mkdir build +cd build + +# Configure the build +cmake [-D CMAKE_INSTALL_PREFIX="path/to/installation"] \ + [-D BUILD_TESTING=False] \ + -D CMAKE_BUILD_TYPE=Release \ + -G <"Unix Makefiles" | "Ninja"> .. + +# install mio + +``` + +To use a dynamic configuration build tool, such as Visual Studio or Xcode: + +```sh +cd +mkdir build +cd build + +# Configure the project +cmake [-D CMAKE_INSTALL_PREFIX="path/to/installation"] \ + [-D BUILD_TESTING=False] \ + -G <"Visual Studio 14 2015 Win64" | "Xcode"> .. + +# install mio +cmake --build . --config Release --target install +``` + +Note that the last command of the installation sequence may require administrator privileges (e.g. `sudo`) if the installation root directory lies outside your home directory. + +This installation ++ copies the mio header files to the `include/mio` subdirectory of the installation root ++ generates and copies several CMake configuration files to the `share/cmake/mio` subdirectory of the installation root + +This latter step allows downstream CMake projects to consume mio via `find_package`, e.g. + +```cmake +find_package( mio REQUIRED ) +target_link_libraries( MyTarget PUBLIC mio::mio ) +``` + +**WINDOWS USERS**: The `mio::mio` target `#define`s `WIN32_LEAN_AND_MEAN` and `NOMINMAX`. The former ensures the imported surface area of the Win API is minimal, and the latter disables Windows' `min` and `max` macros so they don't intefere with `std::min` and `std::max`. Because *mio* is a header only library, these defintions will leak into downstream CMake builds. If their presence is causing problems with your build then you can use the alternative `mio::mio_full_winapi` target, which adds none of these defintions. + +If mio was installed to a non-conventional location, it may be necessary for downstream projects to specify the mio installation root directory via either + ++ the `CMAKE_PREFIX_PATH` configuration option, ++ the `CMAKE_PREFIX_PATH` environment variable, or ++ `mio_DIR` environment variable. + +Please see the [Kitware documentation](https://cmake.org/cmake/help/v3.0/command/find_package.html) for more information. + +In addition, mio supports packaged relocatable installations via [CPack](https://cmake.org/cmake/help/latest/manual/cpack.1.html). +Following configuration, from the build directory, invoke cpack as follows to generate a packaged installation: + +```sh +cpack -G -C Release +``` + +The list of supported generators varies from platform to platform. See the output of `cpack --help` for a complete list of supported generators on your platform. + +### Subproject Composition +To use mio as a subproject, copy the mio repository to your project's dependencies/externals folder. +If your project is version controlled using git, a git submodule or git subtree can be used to syncronize with the updstream repository. +The [use](https://services.github.com/on-demand/downloads/submodule-vs-subtree-cheat-sheet/) and [relative advantages](https://andrey.nering.com.br/2016/git-submodules-vs-subtrees/) of these git facilities is beyond the scope of this document, but in brief, each may be established as follows: + +```sh +# via git submodule +cd +git submodule add -b master https://github.com/mandreyel/mio.git + +# via git subtree +cd +git subtree add --prefix /mio \ + https://github.com/mandreyel/mio.git master --squash +``` + +Given a mio subdirectory in a project, simply add the following lines to your project's to add mio include directories to your target's include path. + +```cmake +add_subdirectory( path/to/mio/ ) +target_link_libraries( MyTarget PUBLIC ) +``` + +Note that, as a subproject, mio's tests and examples will not be built and CPack integration is deferred to the host project. + diff --git a/src/mio/detail/mmap.ipp b/src/mio/detail/mmap.ipp new file mode 100644 index 0000000..716a171 --- /dev/null +++ b/src/mio/detail/mmap.ipp @@ -0,0 +1,533 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_BASIC_MMAP_IMPL +#define MIO_BASIC_MMAP_IMPL + +#include "mio/mmap.hpp" +#include "mio/page.hpp" +#include "mio/detail/string_util.hpp" + +#include + +#ifndef _WIN32 +# include +# include +# include +# include +#endif + +namespace mio { +namespace detail { + +#ifdef _WIN32 +namespace win { + +/** Returns the 4 upper bytes of an 8-byte integer. */ +inline DWORD int64_high(int64_t n) noexcept +{ + return n >> 32; +} + +/** Returns the 4 lower bytes of an 8-byte integer. */ +inline DWORD int64_low(int64_t n) noexcept +{ + return n & 0xffffffff; +} + +inline std::wstring s_2_ws(const std::string& s) +{ + std::wstring ret; + if (!s.empty()) + { + ret.resize(s.size()); + int wide_char_count = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), + static_cast(s.size()), &ret[0], static_cast(s.size())); + ret.resize(wide_char_count); + } + return ret; +} + +template< + typename String, + typename = typename std::enable_if< + std::is_same::type, char>::value + >::type +> file_handle_type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(s_2_ws(path).c_str(), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +template +typename std::enable_if< + std::is_same::type, wchar_t>::value, + file_handle_type +>::type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(c_str(path), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +} // win +#endif // _WIN32 + +/** + * Returns the last platform specific system error (errno on POSIX and + * GetLastError on Win) as a `std::error_code`. + */ +inline std::error_code last_error() noexcept +{ + std::error_code error; +#ifdef _WIN32 + error.assign(GetLastError(), std::system_category()); +#else + error.assign(errno, std::system_category()); +#endif + return error; +} + +template +file_handle_type open_file(const String& path, const access_mode mode, + std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return invalid_handle; + } +#ifdef _WIN32 + const auto handle = win::open_file_helper(path, mode); +#else // POSIX + const auto handle = ::open(c_str(path), + mode == access_mode::read ? O_RDONLY : O_RDWR); +#endif + if(handle == invalid_handle) + { + error = detail::last_error(); + } + return handle; +} + +inline size_t query_file_size(file_handle_type handle, std::error_code& error) +{ + error.clear(); +#ifdef _WIN32 + LARGE_INTEGER file_size; + if(::GetFileSizeEx(handle, &file_size) == 0) + { + error = detail::last_error(); + return 0; + } + return static_cast(file_size.QuadPart); +#else // POSIX + struct stat sbuf; + if(::fstat(handle, &sbuf) == -1) + { + error = detail::last_error(); + return 0; + } + return sbuf.st_size; +#endif +} + +struct mmap_context +{ + char* data; + int64_t length; + int64_t mapped_length; +#ifdef _WIN32 + file_handle_type file_mapping_handle; +#endif +}; + +inline mmap_context memory_map(const file_handle_type file_handle, const int64_t offset, + const int64_t length, const access_mode mode, std::error_code& error) +{ + const int64_t aligned_offset = make_offset_page_aligned(offset); + const int64_t length_to_map = offset - aligned_offset + length; +#ifdef _WIN32 + const int64_t max_file_size = offset + length; + const auto file_mapping_handle = ::CreateFileMapping( + file_handle, + 0, + mode == access_mode::read ? PAGE_READONLY : PAGE_READWRITE, + win::int64_high(max_file_size), + win::int64_low(max_file_size), + 0); + if(file_mapping_handle == invalid_handle) + { + error = detail::last_error(); + return {}; + } + char* mapping_start = static_cast(::MapViewOfFile( + file_mapping_handle, + mode == access_mode::read ? FILE_MAP_READ : FILE_MAP_WRITE, + win::int64_high(aligned_offset), + win::int64_low(aligned_offset), + length_to_map)); + if(mapping_start == nullptr) + { + // Close file handle if mapping it failed. + ::CloseHandle(file_mapping_handle); + error = detail::last_error(); + return {}; + } +#else // POSIX + char* mapping_start = static_cast(::mmap( + 0, // Don't give hint as to where to map. + length_to_map, + mode == access_mode::read ? PROT_READ : PROT_WRITE, + MAP_SHARED, + file_handle, + aligned_offset)); + if(mapping_start == MAP_FAILED) + { + error = detail::last_error(); + return {}; + } +#endif + mmap_context ctx; + ctx.data = mapping_start + offset - aligned_offset; + ctx.length = length; + ctx.mapped_length = length_to_map; +#ifdef _WIN32 + ctx.file_mapping_handle = file_mapping_handle; +#endif + return ctx; +} + +} // namespace detail + +// -- basic_mmap -- + +template +basic_mmap::~basic_mmap() +{ + conditional_sync(); + unmap(); +} + +template +basic_mmap::basic_mmap(basic_mmap&& other) + : data_(std::move(other.data_)) + , length_(std::move(other.length_)) + , mapped_length_(std::move(other.mapped_length_)) + , file_handle_(std::move(other.file_handle_)) +#ifdef _WIN32 + , file_mapping_handle_(std::move(other.file_mapping_handle_)) +#endif + , is_handle_internal_(std::move(other.is_handle_internal_)) +{ + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif +} + +template +basic_mmap& +basic_mmap::operator=(basic_mmap&& other) +{ + if(this != &other) + { + // First the existing mapping needs to be removed. + unmap(); + data_ = std::move(other.data_); + length_ = std::move(other.length_); + mapped_length_ = std::move(other.mapped_length_); + file_handle_ = std::move(other.file_handle_); +#ifdef _WIN32 + file_mapping_handle_ = std::move(other.file_mapping_handle_); +#endif + is_handle_internal_ = std::move(other.is_handle_internal_); + + // The moved from basic_mmap's fields need to be reset, because + // otherwise other's destructor will unmap the same mapping that was + // just moved into this. + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif + other.is_handle_internal_ = false; + } + return *this; +} + +template +typename basic_mmap::handle_type +basic_mmap::mapping_handle() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_; +#else + return file_handle_; +#endif +} + +template +template +void basic_mmap::map(const String& path, const size_type offset, + const size_type length, std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + const auto handle = detail::open_file(path, AccessMode, error); + if(error) + { + return; + } + + map(handle, offset, length, error); + // This MUST be after the call to map, as that sets this to true. + if(!error) + { + is_handle_internal_ = true; + } +} + +template +void basic_mmap::map(const handle_type handle, + const size_type offset, const size_type length, std::error_code& error) +{ + error.clear(); + if(handle == invalid_handle) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + const auto file_size = detail::query_file_size(handle, error); + if(error) + { + return; + } + + if(offset + length > file_size) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + + const auto ctx = detail::memory_map(handle, offset, + length == map_entire_file ? (file_size - offset) : length, + AccessMode, error); + if(!error) + { + // We must unmap the previous mapping that may have existed prior to this call. + // Note that this must only be invoked after a new mapping has been created in + // order to provide the strong guarantee that, should the new mapping fail, the + // `map` function leaves this instance in a state as though the function had + // never been invoked. + unmap(); + file_handle_ = handle; + is_handle_internal_ = false; + data_ = reinterpret_cast(ctx.data); + length_ = ctx.length; + mapped_length_ = ctx.mapped_length; +#ifdef _WIN32 + file_mapping_handle_ = ctx.file_mapping_handle; +#endif + } +} + +template +template +typename std::enable_if::type +basic_mmap::sync(std::error_code& error) +{ + error.clear(); + if(!is_open()) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + if(data()) + { +#ifdef _WIN32 + if(::FlushViewOfFile(get_mapping_start(), mapped_length_) == 0 + || ::FlushFileBuffers(file_handle_) == 0) +#else // POSIX + if(::msync(get_mapping_start(), mapped_length_, MS_SYNC) != 0) +#endif + { + error = detail::last_error(); + return; + } + } +#ifdef _WIN32 + if(::FlushFileBuffers(file_handle_) == 0) + { + error = detail::last_error(); + } +#endif +} + +template +void basic_mmap::unmap() +{ + if(!is_open()) { return; } + // TODO do we care about errors here? +#ifdef _WIN32 + if(is_mapped()) + { + ::UnmapViewOfFile(get_mapping_start()); + ::CloseHandle(file_mapping_handle_); + } +#else // POSIX + if(data_) { ::munmap(const_cast(get_mapping_start()), mapped_length_); } +#endif + + // If `file_handle_` was obtained by our opening it (when map is called with + // a path, rather than an existing file handle), we need to close it, + // otherwise it must not be closed as it may still be used outside this + // instance. + if(is_handle_internal_) + { +#ifdef _WIN32 + ::CloseHandle(file_handle_); +#else // POSIX + ::close(file_handle_); +#endif + } + + // Reset fields to their default values. + data_ = nullptr; + length_ = mapped_length_ = 0; + file_handle_ = invalid_handle; +#ifdef _WIN32 + file_mapping_handle_ = invalid_handle; +#endif +} + +template +bool basic_mmap::is_mapped() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_ != invalid_handle; +#else // POSIX + return is_open(); +#endif +} + +template +void basic_mmap::swap(basic_mmap& other) +{ + if(this != &other) + { + using std::swap; + swap(data_, other.data_); + swap(file_handle_, other.file_handle_); +#ifdef _WIN32 + swap(file_mapping_handle_, other.file_mapping_handle_); +#endif + swap(length_, other.length_); + swap(mapped_length_, other.mapped_length_); + swap(is_handle_internal_, other.is_handle_internal_); + } +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // This is invoked from the destructor, so not much we can do about + // failures here. + std::error_code ec; + sync(ec); +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // noop +} + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b) +{ + return a.data() == b.data() + && a.size() == b.size(); +} + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a == b); +} + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() < b.size(); } + return a.data() < b.data(); +} + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a > b); +} + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() > b.size(); } + return a.data() > b.data(); +} + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a < b); +} + +} // namespace mio + +#endif // MIO_BASIC_MMAP_IMPL diff --git a/src/mio/detail/string_util.hpp b/src/mio/detail/string_util.hpp new file mode 100644 index 0000000..2f375aa --- /dev/null +++ b/src/mio/detail/string_util.hpp @@ -0,0 +1,170 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_STRING_UTIL_HEADER +#define MIO_STRING_UTIL_HEADER + +#include + +namespace mio { +namespace detail { + +template< + typename S, + typename C = typename std::decay::type, + typename = decltype(std::declval().data()), + typename = typename std::enable_if< + std::is_same::value +#ifdef _WIN32 + || std::is_same::value +#endif + >::type +> struct char_type_helper { + using type = typename C::value_type; +}; + +template +struct char_type { + using type = typename char_type_helper::type; +}; + +// TODO: can we avoid this brute force approach? +template<> +struct char_type { + using type = char; +}; + +template<> +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +#ifdef _WIN32 +template<> +struct char_type { + using type = wchar_t; +}; + +template<> +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; +#endif // _WIN32 + +template +struct is_c_str_helper +{ + static constexpr bool value = std::is_same< + CharT*, + // TODO: I'm so sorry for this... Can this be made cleaner? + typename std::add_pointer< + typename std::remove_cv< + typename std::remove_pointer< + typename std::decay< + S + >::type + >::type + >::type + >::type + >::value; +}; + +template +struct is_c_str +{ + static constexpr bool value = is_c_str_helper::value; +}; + +#ifdef _WIN32 +template +struct is_c_wstr +{ + static constexpr bool value = is_c_str_helper::value; +}; +#endif // _WIN32 + +template +struct is_c_str_or_c_wstr +{ + static constexpr bool value = is_c_str::value +#ifdef _WIN32 + || is_c_wstr::value +#endif + ; +}; + +template< + typename String, + typename = decltype(std::declval().data()), + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(const String& path) +{ + return path.data(); +} + +template< + typename String, + typename = decltype(std::declval().empty()), + typename = typename std::enable_if::value>::type +> bool empty(const String& path) +{ + return path.empty(); +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(String path) +{ + return path; +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> bool empty(String path) +{ + return !path || (*path == 0); +} + +} // namespace detail +} // namespace mio + +#endif // MIO_STRING_UTIL_HEADER diff --git a/src/mio/mmap.hpp b/src/mio/mmap.hpp new file mode 100644 index 0000000..def559a --- /dev/null +++ b/src/mio/mmap.hpp @@ -0,0 +1,492 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_MMAP_HEADER +#define MIO_MMAP_HEADER + +#include "mio/page.hpp" + +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif // WIN32_LEAN_AND_MEAN +# include +#else // ifdef _WIN32 +# define INVALID_HANDLE_VALUE -1 +#endif // ifdef _WIN32 + +namespace mio { + +// This value may be provided as the `length` parameter to the constructor or +// `map`, in which case a memory mapping of the entire file is created. +enum { map_entire_file = 0 }; + +#ifdef _WIN32 +using file_handle_type = HANDLE; +#else +using file_handle_type = int; +#endif + +// This value represents an invalid file handle type. This can be used to +// determine whether `basic_mmap::file_handle` is valid, for example. +const static file_handle_type invalid_handle = INVALID_HANDLE_VALUE; + +template +struct basic_mmap +{ + using value_type = ByteT; + using size_type = size_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using iterator_category = std::random_access_iterator_tag; + using handle_type = file_handle_type; + + static_assert(sizeof(ByteT) == sizeof(char), "ByteT must be the same size as char."); + +private: + // Points to the first requested byte, and not to the actual start of the mapping. + pointer data_ = nullptr; + + // Length--in bytes--requested by user (which may not be the length of the + // full mapping) and the length of the full mapping. + size_type length_ = 0; + size_type mapped_length_ = 0; + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity (see `is_handle_internal_`). + // On POSIX, we only need a file handle to create a mapping, while on + // Windows systems the file handle is necessary to retrieve a file mapping + // handle, but any subsequent operations on the mapped region must be done + // through the latter. + handle_type file_handle_ = INVALID_HANDLE_VALUE; +#ifdef _WIN32 + handle_type file_mapping_handle_ = INVALID_HANDLE_VALUE; +#endif + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity in that we must not close the file handle if + // user provided it, but we must close it if we obtained it using the + // provided path. For this reason, this flag is used to determine when to + // close `file_handle_`. + bool is_handle_internal_; + +public: + /** + * The default constructed mmap object is in a non-mapped state, that is, + * any operation that attempts to access nonexistent underlying data will + * result in undefined behaviour/segmentation faults. + */ + basic_mmap() = default; + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * `basic_mmap` has single-ownership semantics, so transferring ownership + * may only be accomplished by moving the object. + */ + basic_mmap(const basic_mmap&) = delete; + basic_mmap(basic_mmap&&); + basic_mmap& operator=(const basic_mmap&) = delete; + basic_mmap& operator=(basic_mmap&&); + + /** + * If this is a read-write mapping, the destructor invokes sync. Regardless + * of the access mode, unmap is invoked as a final step. + */ + ~basic_mmap(); + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept { return file_handle_; } + handle_type mapping_handle() const noexcept; + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return file_handle_ != invalid_handle; } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return length() == 0; } + + /** Returns true if a mapping was established. */ + bool is_mapped() const noexcept; + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return length(); } + size_type length() const noexcept { return length_; } + size_type mapped_length() const noexcept { return mapped_length_; } + + /** Returns the offset relative to the start of the mapping. */ + size_type mapping_offset() const noexcept + { + return mapped_length_ - length_; + } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return data_; } + const_pointer data() const noexcept { return data_; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator begin() noexcept { return data(); } + const_iterator begin() const noexcept { return data(); } + const_iterator cbegin() const noexcept { return data(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return data() + length(); } + const_iterator end() const noexcept { return data() + length(); } + const_iterator cend() const noexcept { return data() + length(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const noexcept + { return const_reverse_iterator(end()); } + const_reverse_iterator crbegin() const noexcept + { return const_reverse_iterator(end()); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return reverse_iterator(begin()); } + const_reverse_iterator rend() const noexcept + { return const_reverse_iterator(begin()); } + const_reverse_iterator crend() const noexcept + { return const_reverse_iterator(begin()); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return data_[i]; } + const_reference operator[](const size_type i) const noexcept { return data_[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap(); + + void swap(basic_mmap& other); + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template + typename std::enable_if::type + sync(std::error_code& error); + + /** + * All operators compare the address of the first byte and size of the two mapped + * regions. + */ + +private: + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer get_mapping_start() noexcept + { + return !data() ? nullptr : data() - mapping_offset(); + } + + const_pointer get_mapping_start() const noexcept + { + return !data() ? nullptr : data() - mapping_offset(); + } + + /** + * The destructor syncs changes to disk if `AccessMode` is `write`, but not + * if it's `read`, but since the destructor cannot be templated, we need to + * do SFINAE in a dedicated function, where one syncs and the other is a noop. + */ + template + typename std::enable_if::type + conditional_sync(); + template + typename std::enable_if::type conditional_sync(); +}; + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b); + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_source = basic_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_sink = basic_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using mmap_source = basic_mmap_source; +using ummap_source = basic_mmap_source; + +using mmap_sink = basic_mmap_sink; +using ummap_sink = basic_mmap_sink; + +/** + * Convenience factory method that constructs a mapping for any `basic_mmap` or + * `basic_mmap` type. + */ +template< + typename MMap, + typename MappingToken +> MMap make_mmap(const MappingToken& token, + int64_t offset, int64_t length, std::error_code& error) +{ + MMap mmap; + mmap.map(token, offset, length, error); + return mmap; +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_source::handle_type`. + */ +template +mmap_source make_mmap_source(const MappingToken& token, mmap_source::size_type offset, + mmap_source::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_source make_mmap_source(const MappingToken& token, std::error_code& error) +{ + return make_mmap_source(token, 0, map_entire_file, error); +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_sink::handle_type`. + */ +template +mmap_sink make_mmap_sink(const MappingToken& token, mmap_sink::size_type offset, + mmap_sink::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_sink make_mmap_sink(const MappingToken& token, std::error_code& error) +{ + return make_mmap_sink(token, 0, map_entire_file, error); +} + +} // namespace mio + +#include "detail/mmap.ipp" + +#endif // MIO_MMAP_HEADER diff --git a/src/mio/page.hpp b/src/mio/page.hpp new file mode 100644 index 0000000..cae7377 --- /dev/null +++ b/src/mio/page.hpp @@ -0,0 +1,78 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_PAGE_HEADER +#define MIO_PAGE_HEADER + +#ifdef _WIN32 +# include +#else +# include +#endif + +namespace mio { + +/** + * This is used by `basic_mmap` to determine whether to create a read-only or + * a read-write memory mapping. + */ +enum class access_mode +{ + read, + write +}; + +/** + * Determines the operating system's page allocation granularity. + * + * On the first call to this function, it invokes the operating system specific syscall + * to determine the page size, caches the value, and returns it. Any subsequent call to + * this function serves the cached value, so no further syscalls are made. + */ +inline size_t page_size() +{ + static const size_t page_size = [] + { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return SystemInfo.dwAllocationGranularity; +#else + return sysconf(_SC_PAGE_SIZE); +#endif + }(); + return page_size; +} + +/** + * Alligns `offset` to the operating's system page size such that it subtracts the + * difference until the nearest page boundary before `offset`, or does nothing if + * `offset` is already page aligned. + */ +inline size_t make_offset_page_aligned(size_t offset) noexcept +{ + const size_t page_size_ = page_size(); + // Use integer division to round down to the nearest page alignment. + return offset / page_size_ * page_size_; +} + +} // namespace mio + +#endif // MIO_PAGE_HEADER diff --git a/src/mio/shared_mmap.hpp b/src/mio/shared_mmap.hpp new file mode 100644 index 0000000..f125a59 --- /dev/null +++ b/src/mio/shared_mmap.hpp @@ -0,0 +1,406 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_SHARED_MMAP_HEADER +#define MIO_SHARED_MMAP_HEADER + +#include "mio/mmap.hpp" + +#include // std::error_code +#include // std::shared_ptr + +namespace mio { + +/** + * Exposes (nearly) the same interface as `basic_mmap`, but endowes it with + * `std::shared_ptr` semantics. + * + * This is not the default behaviour of `basic_mmap` to avoid allocating on the heap if + * shared semantics are not required. + */ +template< + access_mode AccessMode, + typename ByteT +> class basic_shared_mmap +{ + using impl_type = basic_mmap; + std::shared_ptr pimpl_; + +public: + using value_type = typename impl_type::value_type; + using size_type = typename impl_type::size_type; + using reference = typename impl_type::reference; + using const_reference = typename impl_type::const_reference; + using pointer = typename impl_type::pointer; + using const_pointer = typename impl_type::const_pointer; + using difference_type = typename impl_type::difference_type; + using iterator = typename impl_type::iterator; + using const_iterator = typename impl_type::const_iterator; + using reverse_iterator = typename impl_type::reverse_iterator; + using const_reverse_iterator = typename impl_type::const_reverse_iterator; + using iterator_category = typename impl_type::iterator_category; + using handle_type = typename impl_type::handle_type; + using mmap_type = impl_type; + + basic_shared_mmap() = default; + basic_shared_mmap(const basic_shared_mmap&) = default; + basic_shared_mmap& operator=(const basic_shared_mmap&) = default; + basic_shared_mmap(basic_shared_mmap&&) = default; + basic_shared_mmap& operator=(basic_shared_mmap&&) = default; + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap(mmap_type&& mmap) + : pimpl_(std::make_shared(std::move(mmap))) + {} + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap& operator=(mmap_type&& mmap) + { + pimpl_ = std::make_shared(std::move(mmap)); + return *this; + } + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap(std::shared_ptr mmap) : pimpl_(std::move(mmap)) {} + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap& operator=(std::shared_ptr mmap) + { + pimpl_ = std::move(mmap); + return *this; + } + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_shared_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_shared_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * If this is a read-write mapping and the last reference to the mapping, + * the destructor invokes sync. Regardless of the access mode, unmap is + * invoked as a final step. + */ + ~basic_shared_mmap() = default; + + /** Returns the underlying `std::shared_ptr` instance that holds the mmap. */ + std::shared_ptr get_shared_ptr() { return pimpl_; } + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept + { + return pimpl_ ? pimpl_->file_handle() : invalid_handle; + } + + handle_type mapping_handle() const noexcept + { + return pimpl_ ? pimpl_->mapping_handle() : invalid_handle; + } + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return pimpl_ && pimpl_->is_open(); } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return !pimpl_ || pimpl_->empty(); } + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type length() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type mapped_length() const noexcept + { + return pimpl_ ? pimpl_->mapped_length() : 0; + } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return pimpl_->data(); } + const_pointer data() const noexcept { return pimpl_ ? pimpl_->data() : nullptr; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + iterator begin() noexcept { return pimpl_->begin(); } + const_iterator begin() const noexcept { return pimpl_->begin(); } + const_iterator cbegin() const noexcept { return pimpl_->cbegin(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return pimpl_->end(); } + const_iterator end() const noexcept { return pimpl_->end(); } + const_iterator cend() const noexcept { return pimpl_->cend(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return pimpl_->rbegin(); } + const_reverse_iterator rbegin() const noexcept { return pimpl_->rbegin(); } + const_reverse_iterator crbegin() const noexcept { return pimpl_->crbegin(); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return pimpl_->rend(); } + const_reverse_iterator rend() const noexcept { return pimpl_->rend(); } + const_reverse_iterator crend() const noexcept { return pimpl_->crend(); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return (*pimpl_)[i]; } + const_reference operator[](const size_type i) const noexcept { return (*pimpl_)[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(path, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map_impl(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(handle, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map_impl(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap() { if(pimpl_) pimpl_->unmap(); } + + void swap(basic_shared_mmap& other) { pimpl_.swap(other.pimpl_); } + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > void sync(std::error_code& error) { if(pimpl_) pimpl_->sync(error); } + + /** All operators compare the underlying `basic_mmap`'s addresses. */ + + friend bool operator==(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ == b.pimpl_; + } + + friend bool operator!=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return !(a == b); + } + + friend bool operator<(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ < b.pimpl_; + } + + friend bool operator<=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ <= b.pimpl_; + } + + friend bool operator>(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ > b.pimpl_; + } + + friend bool operator>=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ >= b.pimpl_; + } + +private: + template + void map_impl(const MappingToken& token, const size_type offset, + const size_type length, std::error_code& error) + { + if(!pimpl_) + { + mmap_type mmap = make_mmap(token, offset, length, error); + if(error) { return; } + pimpl_ = std::make_shared(std::move(mmap)); + } + else + { + pimpl_->map(token, offset, length, error); + } + } +}; + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_source = basic_shared_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_sink = basic_shared_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using shared_mmap_source = basic_shared_mmap_source; +using shared_ummap_source = basic_shared_mmap_source; + +using shared_mmap_sink = basic_shared_mmap_sink; +using shared_ummap_sink = basic_shared_mmap_sink; + +} // namespace mio + +#endif // MIO_SHARED_MMAP_HEADER diff --git a/src/zip/DeflateFileWriter.cpp b/src/zip/DeflateFileWriter.cpp new file mode 100644 index 0000000..d00752a --- /dev/null +++ b/src/zip/DeflateFileWriter.cpp @@ -0,0 +1,31 @@ +#include "StdAfx.h" + +#include "fs/zip/DeflateFileWriter.h" +#include "fs/IFilesystem.h" +#include "fs/zip/DeflateSinkWriter.h" + +namespace fs +{ + +////////////////////////////////////////////////////////////////////////// + +DeflateFileWriter::DeflateFileWriter(AbsPath i_filePath, tl::lent_ref i_filesystem, uint8_t i_compressionLevel) + : m_filePath(std::move(i_filePath)) + , m_filesystem(std::move(i_filesystem)) + , m_compressionLevel(i_compressionLevel) +{ +} + +////////////////////////////////////////////////////////////////////////// + +ZipWriter::DataWriterResult DeflateFileWriter::operator()(IStreamSink& i_sink) +{ + auto result = m_filesystem->openSource(m_filePath); + if (result.has_error()) + return tl::make_generic_error("Error opening input file '{}' to deflate: {}", m_filePath, result.error()); + return DeflateSinkWriter(*result.value(), m_compressionLevel)(i_sink); +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/zip/DeflateSinkWriter.cpp b/src/zip/DeflateSinkWriter.cpp new file mode 100644 index 0000000..44dea32 --- /dev/null +++ b/src/zip/DeflateSinkWriter.cpp @@ -0,0 +1,60 @@ +#include "StdAfx.h" + +#include "fs/zip/DeflateSinkWriter.h" + +#include "fs/CRCStream.h" +#include "fs/CopyStream.h" +#include "fs/IStreamSink.h" +#include "fs/DeflateStreamSink.h" +#include "fs/ISource.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +DeflateSinkWriter::DeflateSinkWriter(IStreamSource& source, uint8_t compressionLevel) + : m_source(&source) + , m_compressionLevel(compressionLevel) +{ +} + +////////////////////////////////////////////////////////////////////////// + +DeflateSinkWriter::DeflateSinkWriter(tl::span data, uint8_t compressionLevel) + : m_data(data) + , m_compressionLevel(compressionLevel) +{ +} + +////////////////////////////////////////////////////////////////////////// + +ZipWriter::DataWriterResult DeflateSinkWriter::operator()(IStreamSink& sink) +{ + DeflateStreamSink deflateSink(sink); + deflateSink.setCompressionLevel(m_compressionLevel); + + if (m_source) + { + uint32_t crc = computeCRC32(0, {}); //zip precondition + auto processFunc = [&crc](tl::span data) + { + crc = computeCRC32(crc, data); + }; + + CopyStreamResult copyResult = copyStream(deflateSink, *m_source, processFunc); + if (copyResult.has_error()) + return tl::make_generic_error("{}", copyResult.error()); + + return ZipWriter::DataWriterPayload{ZipBase::CompressionMethod::Deflate, crc, copyResult.value()}; + } + else + { + uint32_t crc = computeCRC32(0, {}); //zip precondition + crc = computeCRC32(crc, m_data); + deflateSink.write(m_data); + return ZipWriter::DataWriterPayload{ZipBase::CompressionMethod::Deflate, crc, m_data.size()}; + } +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/zip/StoreFileWriter.cpp b/src/zip/StoreFileWriter.cpp new file mode 100644 index 0000000..8e2d2f4 --- /dev/null +++ b/src/zip/StoreFileWriter.cpp @@ -0,0 +1,33 @@ +#include "StdAfx.h" + +#include "fs/zip/StoreFileWriter.h" + +#include "fs/IFilesystem.h" +#include "fs/zip/StoreSinkWriter.h" + +namespace fs +{ + +////////////////////////////////////////////////////////////////////////// + +StoreFileWriter::StoreFileWriter(AbsPath i_filePath, tl::lent_ref i_filesystem) + : m_filePath(std::move(i_filePath)) + , m_filesystem(std::move(i_filesystem)) +{ +} + +////////////////////////////////////////////////////////////////////////// + +ZipWriter::DataWriterResult StoreFileWriter::operator()(IStreamSink& i_sink) +{ + auto openResult = m_filesystem->openSource(m_filePath); + if (openResult.has_error()) + return tl::make_generic_error("Error opening input file '{}' to store: {}", m_filePath, openResult.error()); + + const tl::unique_ref inputSource = std::move(openResult.value()); + return StoreSinkWriter(*inputSource)(i_sink); +} + +////////////////////////////////////////////////////////////////////////// + +} diff --git a/src/zip/StoreSinkWriter.cpp b/src/zip/StoreSinkWriter.cpp new file mode 100644 index 0000000..3f44d1b --- /dev/null +++ b/src/zip/StoreSinkWriter.cpp @@ -0,0 +1,40 @@ +#include "StdAfx.h" + +#include "fs/zip/StoreSinkWriter.h" + +#include "fs/CRCStream.h" +#include "fs/CopyStream.h" +#include "fs/IStreamSink.h" +#include "fs/DeflateStreamSink.h" +#include "fs/ISource.h" + +#include "Logger.h" + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +StoreSinkWriter::StoreSinkWriter(ISource& source) + : m_inSource(source) +{ +} + +////////////////////////////////////////////////////////////////////////// + +ZipWriter::DataWriterResult StoreSinkWriter::operator()(IStreamSink& sink) +{ + uint32_t crc = computeCRC32(0, {}); //zip precondition + auto processFunc = [&crc](tl::span data) + { + crc = computeCRC32(crc, data); + }; + + auto copyRes = copyStream(sink, m_inSource.get(), processFunc); + if (copyRes.has_error()) + return tl::make_generic_error("{}", copyRes.error()); + + return ZipWriter::DataWriterPayload{ZipBase::CompressionMethod::Store, crc, copyRes.value()}; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/zip/ZipBase.cpp b/src/zip/ZipBase.cpp new file mode 100644 index 0000000..71278b9 --- /dev/null +++ b/src/zip/ZipBase.cpp @@ -0,0 +1,59 @@ +#include "StdAfx.h" +#include "fs/zip/ZipBase.h" +#include + +namespace fs +{ + +const uint32_t ZipBase::LocalFileHeader::k_signature; +const uint32_t ZipBase::CentralDirectoryFileHeader::k_signature; +const uint32_t ZipBase::EndOfCentralDirectoryRecord::k_signature; +const uint32_t ZipBase::ExtendedInformationExtraField64::k_tag; +const uint32_t ZipBase::EndOfCentralDirectoryRecord64::k_signature; +const uint32_t ZipBase::EndOfCentralDirectoryLocator64::k_signature; + +////////////////////////////////////////////////////////////////////////// + +void ZipBase::computeMsDosTime(time_t time, uint16_t& o_date, uint16_t& o_time) +{ +#ifdef TL_PLATFORM_WINDOWS_FAMILY + struct tm tm_struct; + struct tm *tm = &tm_struct; + if (errno_t err = localtime_s(tm, &time)) + { + o_date = 0; + o_time = 0; + return; + } +#else + struct tm *tm = std::localtime(&time); +#endif /* #ifdef _MSC_VER */ + + o_time = (uint16_t)(((tm->tm_hour) << 11) + ((tm->tm_min) << 5) + ((tm->tm_sec) >> 1)); + o_date = (uint16_t)(((tm->tm_year + 1900 - 1980) << 9) + ((tm->tm_mon + 1) << 5) + tm->tm_mday); +} + +////////////////////////////////////////////////////////////////////////// + +ZipBase::CentralDirectoryFileHeader ZipBase::computeCentralDirectoryFileHeader(const LocalFileHeader& header, bool zip64) +{ + CentralDirectoryFileHeader centralHeader; + centralHeader.versionMadeBy = 0; + centralHeader.versionNeededToExtract = header.versionNeededToExtract; + centralHeader.generalBitFlag = header.generalBitFlag; + centralHeader.compressionMethod = header.compressionMethod; + centralHeader.lastModFileTime = header.lastModFileTime; + centralHeader.lastModFileDate = header.lastModFileDate; + centralHeader.crc32 = header.crc32; + centralHeader.compressedSize = header.compressedSize; + centralHeader.uncompressedSize = header.uncompressedSize; + centralHeader.filenameLength = header.filenameLength; + centralHeader.extraFieldLength = header.extraFieldLength; + centralHeader.diskNumberStart = zip64 ? static_cast(-1) : 0; + centralHeader.internalFileAttributes = 0; + centralHeader.externalFileAttributes = 0; + return centralHeader; +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/zip/ZipPack.cpp b/src/zip/ZipPack.cpp new file mode 100644 index 0000000..5b3b3c9 --- /dev/null +++ b/src/zip/ZipPack.cpp @@ -0,0 +1,662 @@ +#include "StdAfx.h" + +#include "fs/zip/ZipPack.h" + +#include "fs/MapSourceView.h" +#include "fs/MemorySource.h" +#include "fs/NativeFilesystem.h" +#include "fs/StreamSourceToSourceAdapter.h" +#include "fs/SourceView.h" +#include "fs/DeflateStreamSource.h" + +#include "tl/crash.h" + +#include "Logger.h" + +#include + +#include "tl/narrow_cast.h" + + +////////////////////////////////////////////////////////////////////////// + +namespace fs +{ +namespace +{ +uint64_t computePersistentHash(const AbsPath& path) +{ + uint64_t hash = 0; + for (const tl::string& e : path) + tl::hash_and_combine(hash, e); + return hash; +} +} + +////////////////////////////////////////////////////////////////////////// + +ZipPack::ZipPack(tl::unique_ref zipFileSource, ZipReader zipReader) + : m_zipMapSource(std::move(zipFileSource)) +{ + TL_ASSERT(!m_zipFileLocation.is_valid()); + TL_ASSERT(m_zipMapSource); + createEntries(std::move(zipReader)); +} + +////////////////////////////////////////////////////////////////////////// + +ZipPack::ZipPack(tl::lent_ref filesystem, AbsPath zipFileLocation, ZipReader zipReader) + : m_filesystem(std::move(filesystem)) + , m_zipFileLocation(std::move(zipFileLocation)) +{ + TL_ASSERT(m_zipFileLocation.is_valid()); + TL_ASSERT(m_filesystem); + TL_ASSERT(!m_zipMapSource); + createEntries(std::move(zipReader)); +} + +////////////////////////////////////////////////////////////////////////// + +ZipPack::CreateResult ZipPack::create(tl::unique_ref zipFileSource) +{ + ZipReader::CreateResult zipResult = ZipReader::create(*zipFileSource); + if (zipResult.has_error()) + return tl::make_error(Error::code_t::SystemError, "Faild to create zip reader: {}", zipResult.error()); + + ZipReader zipReader = std::move(zipResult.value()); + if (zipReader.getEntryCount() > 65535) + return tl::make_error(Error::code_t::NotSupported, + "ZipPack failed to initialize because the zip has too many entries ({} > 65536 max)", + zipReader.getEntryCount()); + + return tl::unique_ref(new ZipPack(std::move(zipFileSource), std::move(zipReader))); +} + +////////////////////////////////////////////////////////////////////////// + +ZipPack::CreateResult ZipPack::create(tl::lent_ref filesystem, AbsPath zipFileLocation) +{ + tl::result, Error> openResult = filesystem->openSource(zipFileLocation); + if (openResult.has_error()) + return openResult.error(); + + const tl::unique_ref source = std::move(openResult.value()); + + ZipReader::CreateResult zipResult = ZipReader::create(*source); + if (zipResult.has_error()) + return tl::make_error(Error::code_t::SystemError, "Faild to create zip reader: {}", zipResult.error()); + + ZipReader zipReader = std::move(zipResult.value()); + if (zipReader.getEntryCount() > 65535) + return tl::make_error(Error::code_t::NotSupported, + "ZipPack failed to initialize because the zip has too many entries ({} > 65536 max)", + zipReader.getEntryCount()); + + return tl::unique_ref(new ZipPack(std::move(filesystem), std::move(zipFileLocation), std::move(zipReader))); +} + +////////////////////////////////////////////////////////////////////////// + +void ZipPack::createEntries(ZipReader zipReader) +{ + m_files.clear(); + m_folders.clear(); + m_pathToIndex.clear(); + m_childrenIndices.clear(); + + const size_t readerEntryCount = zipReader.getEntryCount(); + + m_files.reserve(readerEntryCount + 1); + m_folders.reserve(readerEntryCount / 2); + m_pathToIndex.reserve(readerEntryCount + 1); + + // add the root + m_folders.emplace_back(); + m_pathToIndex.emplace(computePersistentHash(AbsPath("/")), EntryIndex(true, 0)); + + tl::vector> childrenIndices; + childrenIndices.reserve(readerEntryCount + 1); + + AbsPath path; + + for (size_t readerEntryIndex = 0; readerEntryIndex < readerEntryCount; readerEntryIndex++) + { + const ZipReader::Entry& readerEntry = zipReader.getEntry(readerEntryIndex); + if (readerEntry.compressionMethod != ZipReader::CompressionMethod::Store && readerEntry.compressionMethod != ZipReader::CompressionMethod::Deflate) + { + LOGE("ZipPack entry '{}' because it uses an unsupported compression", readerEntry.name); + continue; + } + if (readerEntry.generalBitFlags.test(ZipReader::GeneralBitFlag::Encrypted)) + { + LOGE("ZipPack entry '{}' because it uses encryption", readerEntry.name); + continue; + } + if ((readerEntry.uncompressedSize >> 32) > 255) + { + LOGE("ZipPack entry '{}' because its uncompressedSize is too big ({} > {} max)", + readerEntry.name, + readerEntry.uncompressedSize, + (uint64_t(255) << 32) | uint64_t(0xFFFFFFFF)); + continue; + } + if ((readerEntry.compressedSize >> 32) > 255) + { + LOGE("ZipPack entry '{}' because its compressedSize is too big ({} > {} max)", + readerEntry.name, + readerEntry.compressedSize, + (uint64_t(255) << 32) | uint64_t(0xFFFFFFFF)); + continue; + } + if ((readerEntry.dataOffset >> 32) > 255) + { + LOGE("ZipPack entry '{}' because its dataOffset is too big ({} > {} max)", + readerEntry.name, + readerEntry.dataOffset, + (uint64_t(255) << 32) | uint64_t(0xFFFFFFFF)); + continue; + } + + bool isFolder = false; + if (!readerEntry.name.empty() && (readerEntry.name.back() == '/' || readerEntry.name.back() == '\\')) + isFolder = true; + + path = PosixRootTag::value(); + path += readerEntry.name; + + AbsPath parentPath = path; + parentPath.pop_back(); + + const uint16_t parentIndex = getOrAddFolder(std::move(parentPath), childrenIndices); + uint16_t index = 0; + + if (isFolder) + { + index = tl::narrow(m_folders.size()); + if (index >= 0xFFFF) + TL_CRASH("Too many folders in zip pack"); + + m_folders.push_back(Folder()); + + Folder& folder = m_folders.back(); + folder.name = path.back(); + folder.parentIndex = parentIndex; + folder.lastModTimePoint = (uint32_t)readerEntry.lastModTimePoint; + } + else + { + index = tl::narrow(m_files.size()); + if (index >= 0xFFFF) + TL_CRASH("Too many files/folders in zip pack"); + + m_files.push_back(File()); + + File& file = m_files.back(); + file.name = path.back(); + file.isCompressed = readerEntry.compressionMethod == ZipReader::CompressionMethod::Deflate; + file.parentIndex = parentIndex; + file.uncompressedSizeH = (uint8_t)(readerEntry.uncompressedSize >> 32); + file.uncompressedSizeL = (uint32_t)(readerEntry.uncompressedSize & 0xFFFFFFFF); + file.compressedSizeH = (uint8_t)(readerEntry.compressedSize >> 32); + file.compressedSizeL = (uint32_t)(readerEntry.compressedSize & 0xFFFFFFFF); + file.dataOffsetH = (uint8_t)(readerEntry.dataOffset >> 32); + file.dataOffsetL = (uint32_t)(readerEntry.dataOffset & 0xFFFFFFFF); + file.lastModTimePoint = (uint32_t)readerEntry.lastModTimePoint; + } + + if (parentIndex >= childrenIndices.size()) + childrenIndices.resize(parentIndex + 1); + + tl::vector& children = childrenIndices[parentIndex]; + children.push_back(EntryIndex(isFolder, index)); + m_pathToIndex.emplace(computePersistentHash(path), EntryIndex(isFolder, index)); + } + + m_childrenIndices.reserve(8192); + + // compact the children indices + for (size_t i = 0; i < childrenIndices.size(); i++) + { + Folder& folder = m_folders[i]; + tl::vector& children = childrenIndices[i]; + folder.childrenStartIndex = (uint16_t)m_childrenIndices.size(); + folder.childrenCount = (uint16_t)children.size(); + if (folder.childrenCount > 0) + { + m_childrenIndices.resize(m_childrenIndices.size() + children.size()); + memcpy(&m_childrenIndices[folder.childrenStartIndex], children.data(), folder.childrenCount * sizeof(EntryIndex)); + } + } + + m_files.shrink_to_fit(); + m_folders.shrink_to_fit(); + m_childrenIndices.shrink_to_fit(); + m_pathToIndex.shrink_to_fit(); +} + +////////////////////////////////////////////////////////////////////////// + +uint16_t ZipPack::getOrAddFolder(AbsPath folderPath, tl::vector>& io_childrenIndices) +{ + if (folderPath.empty()) + return 0; + + uint64_t pathHash = computePersistentHash(folderPath); + const auto it = m_pathToIndex.find(pathHash); + if (it != m_pathToIndex.end()) + { + const EntryIndex& entryIndex = it->second; + if (!entryIndex.isFolder) + TL_CRASH("Inconsistency"); + + return entryIndex.index; + } + + const tl::string name = std::move(folderPath.back()); + AbsPath parentPath = std::move(folderPath); + parentPath.pop_back(); + + const uint16_t parentIndex = getOrAddFolder(parentPath, io_childrenIndices); + + const uint16_t index = tl::narrow(m_folders.size()); + if (index >= 0xFFFF) + TL_CRASH("Too many files/folders in zip pack"); + + m_folders.push_back(Folder()); + Folder& folder = m_folders.back(); + folder.name = name; + folder.parentIndex = parentIndex; + + if (parentIndex >= io_childrenIndices.size()) + io_childrenIndices.resize(parentIndex + 1); + + tl::vector& children = io_childrenIndices[parentIndex]; + children.push_back(EntryIndex(true, index)); + m_pathToIndex.emplace(pathHash, EntryIndex(true, index)); + + return index; +} + +////////////////////////////////////////////////////////////////////////// + +void ZipPack::setEncryptionData(const tl::string& key, uint32_t rounds) +{ + m_encryptionKey = key; + m_encryptionRounds = rounds; +} + +////////////////////////////////////////////////////////////////////////// + +namespace +{ +uint64_t decodeValue(uint8_t h, uint32_t l) +{ + return (uint64_t(h) << 32) | uint64_t(l); +} +} + +////////////////////////////////////////////////////////////////////////// + +OpenSourceResult ZipPack::openSource(const AbsPath& path, IPack::SourceFlags flags) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + return tl::make_error(ErrorCode::NotFound, "Path '{}' doesn't exist", path); + + const EntryIndex& entryIndex = it->second; + if (entryIndex.isFolder) + return tl::make_error(ErrorCode::NotFound, "Trying to open a source on a folder"); + + const File& file = m_files[entryIndex.index]; + const uint64_t uncompressedSize = decodeValue(file.uncompressedSizeH, file.uncompressedSizeL); + if (!file.isCompressed) + { + if (uncompressedSize == 0) + return tl::make_unique(); // we return an empty source + + // Not compressed - the most simple and fast case + return openRawSource(file); + } + + + + return openZipSource(file); +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSourceResult ZipPack::openStreamSource(const AbsPath& path, IPack::SourceFlags flags) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + return tl::make_error(ErrorCode::NotFound, "Path '{}' doesn't exist", path); + + const EntryIndex& entryIndex = it->second; + if (entryIndex.isFolder) + return tl::make_error(ErrorCode::NotFound, "Trying to open a source on a folder"); + + return openZipStreamSource(m_files[entryIndex.index]); +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSourceResult ZipPack::openMapSource(const AbsPath& path, MapView mapView, SourceFlags flags) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + return tl::make_error(ErrorCode::NotFound, "Path '{}' doesn't exist", path); + + const EntryIndex& entryIndex = it->second; + if (entryIndex.isFolder) + return tl::make_error(ErrorCode::NotFound, "Trying to open a source on a folder"); + + const File& file = m_files[entryIndex.index]; + const uint64_t uncompressedSize = decodeValue(file.uncompressedSizeH, file.uncompressedSizeL); + if (!file.isCompressed) + { + if (uncompressedSize == 0) + return tl::make_unique(); // we return an empty source + + // Not compressed - the most simple and fast case + return openRawMapSource(file, mapView); + } + + + return openZipMapSource(file); +} + +////////////////////////////////////////////////////////////////////////// + +ConvertToNativePathResult ZipPack::convertToNativePath(const AbsPath& path) const +{ + return tl::make_error(ErrorCode::NotAllowed, "Cannot convert zip paths to native ones '{}'", path); +} + +////////////////////////////////////////////////////////////////////////// + +IsFileResult ZipPack::isFile(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + return tl::make_error(ErrorCode::NotFound, "Path '{}' doesn't exist", path); + + const EntryIndex& entryIndex = it->second; + return !entryIndex.isFolder; +} + +////////////////////////////////////////////////////////////////////////// + +IsFolderResult ZipPack::isFolder(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + return tl::make_error(ErrorCode::NotFound, "Path '{}' doesn't exist", path); + + const EntryIndex& entryIndex = it->second; + return entryIndex.isFolder; +} + +////////////////////////////////////////////////////////////////////////// + +ExistsResult ZipPack::exists(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + return m_pathToIndex.find(computePersistentHash(path)) != m_pathToIndex.end(); +} + +////////////////////////////////////////////////////////////////////////// + +GetStatResult ZipPack::getStat(const AbsPath& path) const +{ + if (!path.is_valid()) + return tl::make_error(ErrorCode::BadPath, "Invalid Path"); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + return tl::make_error(ErrorCode::NotFound, "Path '{}' doesn't exist", path); + + const EntryIndex& entryIndex = it->second; + if (entryIndex.isFolder) + return tl::make_error(ErrorCode::InvalidArgument, "Cannot stat a folder: '{}'", path); + + const File& file = m_files[entryIndex.index]; + return Stat{ decodeValue(file.uncompressedSizeH, file.uncompressedSizeL), (time_t)file.lastModTimePoint }; +} + +////////////////////////////////////////////////////////////////////////// + +cppcoro::generator ZipPack::enumerate(const AbsPath& path) const +{ + TL_ASSERT(path.is_valid()); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + { + TL_FAIL("Path '{}' doesn't exist", path); + co_return; + } + + const EntryIndex& entryIndex = it->second; + if (!entryIndex.isFolder) + { + TL_FAIL("Path '{}' is not a folder", path); + co_return; + } + + const Folder& folder = m_folders[entryIndex.index]; + for (size_t i = 0; i < folder.childrenCount; i++) + { + const EntryIndex& childEntryIndex = m_childrenIndices[folder.childrenStartIndex + i]; + if (childEntryIndex.isFolder) + { + const Folder& f = m_folders[childEntryIndex.index]; + co_yield{RelPath(f.name), true}; + } + else + { + const File& f = m_files[childEntryIndex.index]; + co_yield{RelPath(f.name), false}; + } + } +} + +////////////////////////////////////////////////////////////////////////// + +void ZipPack::buildUpPath(RelPath& path, const Folder& folder, uint32_t toIndex) const +{ + path = folder.name; + { + const Folder* f = &folder; + while (f->parentIndex != toIndex && f->parentIndex != 0) + { + f = &m_folders[f->parentIndex]; + path.push_front(f->name); + } + } +} + +void ZipPack::buildUpPath(RelPath& path, const File& file, uint32_t toIndex) const +{ + path = file.name; + if (file.parentIndex != toIndex) + { + const Folder* f = &m_folders[file.parentIndex]; + while (true) + { + path.push_front(f->name); + if (f->parentIndex == toIndex || f->parentIndex == 0) + break; + f = &m_folders[f->parentIndex]; + } + } +} + + +cppcoro::generator ZipPack::enumerateRecursively(const AbsPath& path) const +{ + TL_ASSERT(path.is_valid()); + + const auto it = m_pathToIndex.find(computePersistentHash(path)); + if (it == m_pathToIndex.end()) + { + TL_FAIL("Path '{}' doesn't exist", path); + co_return; + } + + const EntryIndex& entryIndex = it->second; + if (!entryIndex.isFolder) + { + TL_FAIL("Path '{}' is not a folder", path); + co_return; + } + + RelPath relPath; + tl::fixed_vector stack; + stack.reserve(64); + stack.push_back(&m_folders[entryIndex.index]); + for (size_t stackIndex = 0; stackIndex < stack.size(); stackIndex++) + { + const Folder* folder = stack[stackIndex]; + for (size_t i = 0; i < folder->childrenCount; i++) + { + const EntryIndex& childEntryIndex = m_childrenIndices[folder->childrenStartIndex + i]; + if (childEntryIndex.isFolder) + { + const Folder& f = m_folders[childEntryIndex.index]; + buildUpPath(relPath, f, entryIndex.index); + co_yield{relPath, true}; + stack.push_back(&f); + } + else + { + const File& f = m_files[childEntryIndex.index]; + buildUpPath(relPath, f, entryIndex.index); + co_yield{relPath, false}; + } + } + } +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSourceResult ZipPack::openRawMapSource(const File& file, MapView mapView) const +{ + const uint64_t compressedSize = decodeValue(file.compressedSizeH, file.compressedSizeL); + if (compressedSize > tl::numeric_limits::max()) + return tl::make_error(ErrorCode::InvalidArgument, + "Cannot map a file {} of size {}. Maximum mappable size is {}", + m_zipFileLocation, + compressedSize, + tl::numeric_limits::max()); + + if (mapView.size == 0) + mapView.size = tl::narrow(compressedSize); + else + mapView.size = tl::min(mapView.size, tl::narrow(compressedSize)); + + const uint64_t dataOffset = decodeValue(file.dataOffsetH, file.dataOffsetL); + mapView.offset = dataOffset + mapView.offset; + + if (m_zipFileLocation.is_valid()) + { + TL_ASSERT(m_filesystem); + tl::result, Error> result = m_filesystem->openMapSource(m_zipFileLocation, mapView); + if (result.has_error()) + return tl::make_error(result.error().code, + "Failed to open mmap source '{}', offset {}, size {}: {}", + m_zipFileLocation, + dataOffset, + compressedSize, + result.error()); + + return std::move(result.value()); + } + + TL_ASSERT(m_zipMapSource); + + std::lock_guard guard(m_zipMapSourceLock); + return tl::make_unique(tl::promote(m_zipMapSource), mapView.offset, mapView.size); +} + +////////////////////////////////////////////////////////////////////////// + +OpenSourceResult ZipPack::openRawSource(const File& file) const +{ + uint64_t compressedSize = decodeValue(file.compressedSizeH, file.compressedSizeL); + uint64_t dataOffset = decodeValue(file.dataOffsetH, file.dataOffsetL); + + if (m_zipFileLocation.is_valid()) + { + TL_ASSERT(m_filesystem); + tl::result, Error> result = m_filesystem->openSource(m_zipFileLocation); + if (result.has_error()) + return result.error(); + + return tl::make_unique(std::move(result.value()), dataOffset, compressedSize); + } + + TL_ASSERT(m_zipMapSource); + + if (compressedSize > tl::numeric_limits::max()) + return tl::make_error(ErrorCode::InvalidArgument, + "Cannot map a file {} of size {}. Maximum mappable size is {}", + m_zipFileLocation, + compressedSize, + tl::numeric_limits::max()); + + std::lock_guard guard(m_zipMapSourceLock); + return tl::make_unique(tl::promote(m_zipMapSource), dataOffset, compressedSize); +} + +////////////////////////////////////////////////////////////////////////// + +OpenMapSourceResult ZipPack::openZipMapSource(const File& file) const +{ + OUTCOME_TRY(auto source, openRawMapSource(file, MapView())); + + if (file.isCompressed) + { + tl::unique_ref cmpSource = tl::make_unique(std::move(source)); + return tl::make_unique(std::move(cmpSource)); + } + + return std::move(source); +} + +////////////////////////////////////////////////////////////////////////// + +OpenSourceResult ZipPack::openZipSource(const File& file) const +{ + OUTCOME_TRY(auto source, openZipMapSource(file)); + return source; +} + +////////////////////////////////////////////////////////////////////////// + +OpenStreamSourceResult ZipPack::openZipStreamSource(const File& file) const +{ + OUTCOME_TRY(auto source, openRawSource(file)); + + if (file.isCompressed) + return tl::make_unique(std::move(source)); + + return std::move(source); +} +} diff --git a/src/zip/ZipReader.cpp b/src/zip/ZipReader.cpp new file mode 100644 index 0000000..d5b9c00 --- /dev/null +++ b/src/zip/ZipReader.cpp @@ -0,0 +1,237 @@ +#include "StdAfx.h" + +#include "fs/zip/ZipReader.h" + +#include "fs/ISource.h" + +// This reader is implemented pretty close to the https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT standard + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +ZipReader::ZipReader(tl::vector entries) + : m_entries(std::move(entries)) +{ +} + +////////////////////////////////////////////////////////////////////////// + +size_t ZipReader::getEntryCount() const +{ + return m_entries.size(); +} + +////////////////////////////////////////////////////////////////////////// + +const ZipReader::Entry& ZipReader::getEntry(size_t index) const +{ + return m_entries[index]; +} + +////////////////////////////////////////////////////////////////////////// + +ZipReader::SeekResult ZipReader::seekSourceToEntryData(const Entry& entry, ISource& o_source) const +{ + o_source.seekRel(entry.localHeaderOffset); + + LocalFileHeader localHeader; + const size_t read = o_source.read({ reinterpret_cast(&localHeader), sizeof(LocalFileHeader) }); + tl::optional optLastError = o_source.getLastError(); + if (read != sizeof(LocalFileHeader) || optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to seek to entry '{}': {}", entry.name, *optLastError); + + o_source.seekRel(localHeader.filenameLength); + o_source.seekRel(localHeader.extraFieldLength); + + optLastError = o_source.getLastError(); + if (optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to seek to entry '{}': {}", entry.name, *optLastError); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +ZipReader::CentralDirectoryResult ZipReader::findCentralDirectoryBounds(ISource& source) +{ + const uint64_t sourceSize = source.getSize(); + + ////////////////////////////////////////////////////////////////////////// + // Try the end of central directory + uint64_t endOfCentralDirectoryRecordStart = sourceSize - sizeof(EndOfCentralDirectoryRecord); + + EndOfCentralDirectoryRecord endOfCentralDirectoryRecord; + source.seekBeg(endOfCentralDirectoryRecordStart); + + source.read({ reinterpret_cast(&endOfCentralDirectoryRecord), sizeof(EndOfCentralDirectoryRecord) }); + if (endOfCentralDirectoryRecord.signature != EndOfCentralDirectoryRecord::k_signature) + return tl::make_error(ErrorCode::BadSignature, "The central directory appears corrupt (bad signature)"); + + ////////////////////////////////////////////////////////////////////////// + // Try the end of central directory locator for zip64 + if (endOfCentralDirectoryRecordStart >= sizeof(EndOfCentralDirectoryLocator64)) + { + const uint64_t endOfCentralDirectoryLocator64Start = endOfCentralDirectoryRecordStart - sizeof(EndOfCentralDirectoryLocator64); + + EndOfCentralDirectoryLocator64 endOfCentralDirectoryLocator64; + source.seekBeg(endOfCentralDirectoryLocator64Start); + + source.read({ reinterpret_cast(&endOfCentralDirectoryLocator64), sizeof(EndOfCentralDirectoryLocator64) }); + if (endOfCentralDirectoryLocator64.signature == EndOfCentralDirectoryLocator64::k_signature && + endOfCentralDirectoryLocator64.endOfCentralDirectoryRecordOffset < endOfCentralDirectoryLocator64Start) + { + // now try the end of central directory 64 + EndOfCentralDirectoryRecord64 endOfCentralDirectoryRecord64; + source.seekBeg(endOfCentralDirectoryLocator64.endOfCentralDirectoryRecordOffset); + + source.read({ reinterpret_cast(&endOfCentralDirectoryRecord64), sizeof(EndOfCentralDirectoryRecord64) }); + if (endOfCentralDirectoryRecord64.signature == EndOfCentralDirectoryRecord64::k_signature) + { + // we definitely are in zip64 mode! + if (endOfCentralDirectoryRecord64.centralDirectoryOffset > endOfCentralDirectoryLocator64.endOfCentralDirectoryRecordOffset) + return tl::make_error(ErrorCode::BadOffset, "The zip64 central directory appears corrupt (bad offset)"); + + return tl::make_pair(endOfCentralDirectoryRecord64.centralDirectoryOffset, endOfCentralDirectoryLocator64.endOfCentralDirectoryRecordOffset); + } + } + } + + if (endOfCentralDirectoryRecord.centralDirectoryOffset > endOfCentralDirectoryRecordStart) + return tl::make_error(ErrorCode::BadOffset, "The central directory appears corrupt (bad offset)"); + + return tl::make_pair(endOfCentralDirectoryRecord.centralDirectoryOffset, endOfCentralDirectoryRecordStart); +} + +////////////////////////////////////////////////////////////////////////// + +ZipReader::CreateResult ZipReader::create(ISource& source) +{ + CentralDirectoryResult centralDirectoryResult = findCentralDirectoryBounds(source); + if (centralDirectoryResult.has_error()) + return tl::make_error(centralDirectoryResult.error().code, "{}", centralDirectoryResult.error()); + + uint64_t centralDirectoryStartOffset = centralDirectoryResult.value().first; + uint64_t centralDirectoryEndOffset = centralDirectoryResult.value().second; + TL_ASSERT(centralDirectoryStartOffset <= centralDirectoryEndOffset); + + tl::vector entries; + entries.reserve(1024); + + tl::memory_buffer stringBuffer; + + source.seekBeg(centralDirectoryStartOffset); + + while (source.tell() < centralDirectoryEndOffset) + { + CentralDirectoryFileHeader fileHeader; + size_t read = source.read({ reinterpret_cast(&fileHeader), sizeof(CentralDirectoryFileHeader) }); + tl::optional optLastError = source.getLastError(); + if (read != sizeof(CentralDirectoryFileHeader) || optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read central directory: {}", *optLastError); + + if (fileHeader.signature != CentralDirectoryFileHeader::k_signature) + { + return tl::make_error( + ErrorCode::CorruptedFile, + "Central directory header signature mismatch. Expected {}, got {}", CentralDirectoryFileHeader::k_signature, fileHeader.signature); + } + + stringBuffer.resize(fileHeader.filenameLength); + read = source.read({ stringBuffer.data(), fileHeader.filenameLength }); + optLastError = source.getLastError(); + if (read != fileHeader.filenameLength || optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read central directory filename: {}", *optLastError); + + Entry entry; + entry.name = tl::string(reinterpret_cast(stringBuffer.data()), reinterpret_cast(stringBuffer.data() + stringBuffer.size())); + entry.compressionMethod = static_cast(fileHeader.compressionMethod); + entry.generalBitFlags = GeneralBitFlags(fileHeader.generalBitFlag); + entry.uncompressedSize = fileHeader.uncompressedSize; + entry.compressedSize = fileHeader.compressedSize; + entry.crc32 = fileHeader.crc32; + entry.localHeaderOffset = fileHeader.localHeaderOffset; + + { + struct tm parts; + parts.tm_sec = (fileHeader.lastModFileTime & 0x001f) << 1; + parts.tm_min = (fileHeader.lastModFileTime & 0x07e0) >> 5; + parts.tm_hour = (fileHeader.lastModFileTime & 0xf800) >> 11; + parts.tm_mday = (fileHeader.lastModFileDate & 0x001f); + parts.tm_mon = ((fileHeader.lastModFileDate & 0x01e0) >> 5) - 1; + parts.tm_year = ((fileHeader.lastModFileDate & 0xfe00) >> 9) + 80; + parts.tm_wday = parts.tm_yday = 0; + parts.tm_isdst = -1; // DST info "not available" + entry.lastModTimePoint = mktime(&parts); + } + + // check for extended info for zip64 formats + if (fileHeader.extraFieldLength >= sizeof(ExtendedInformationExtraField64) - 4) + { + // the ExtendedInformationExtraField64 is the first one of the extended fields + ExtendedInformationExtraField64 extended; + read = source.read({ reinterpret_cast(&extended), sizeof(ExtendedInformationExtraField64) }); + optLastError = source.getLastError(); + if (read != sizeof(ExtendedInformationExtraField64) || optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read central directory extra fields: {}", *optLastError); + + if (extended.tag == ExtendedInformationExtraField64::k_tag && extended.size == (sizeof(ExtendedInformationExtraField64)- 4)) + { + entry.uncompressedSize = extended.originalSize; + entry.compressedSize = extended.compressedSize; + entry.localHeaderOffset = extended.localHeaderOffset; + } + else + { + // not the tag we're looking for, skip the remaining extra fields + source.seekRel(fileHeader.extraFieldLength - sizeof(ExtendedInformationExtraField64)); + optLastError = source.getLastError(); + if (optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read central directory extra fields: {}", *optLastError); + } + } + else + { + source.seekRel(fileHeader.extraFieldLength); + optLastError = source.getLastError(); + if (optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read central directory extra fields: {}", *optLastError); + } + + if (fileHeader.fileCommentLength > 0) + { + entry.comment.resize(fileHeader.fileCommentLength); + read = source.read({ entry.comment.data(), fileHeader.fileCommentLength }); + optLastError = source.getLastError(); + if (read != fileHeader.fileCommentLength || optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read central directory filename: {}", *optLastError); + } + + entries.push_back(std::move(entry)); + } + + // now go through all the entries, read the local header to calculate the precise data offset + for (Entry& entry : entries) + { + source.seekBeg(entry.localHeaderOffset); + tl::optional optLastError = source.getLastError(); + if (optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to seek to local file header for entry '{}': {}", entry.name, *optLastError); + + LocalFileHeader fileHeader; + size_t read = source.read({ reinterpret_cast(&fileHeader), sizeof(LocalFileHeader) }); + if (read != sizeof(LocalFileHeader)) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read local file header for entry '{}': insufficient data", entry.name); + + optLastError = source.getLastError(); + if (optLastError.has_value()) + return tl::make_error(ErrorCode::CorruptedFile, "Failed to read local file header for entry '{}': {}", entry.name, *optLastError); + + entry.dataOffset = entry.localHeaderOffset + sizeof(LocalFileHeader) + fileHeader.filenameLength + fileHeader.extraFieldLength; + } + + return ZipReader(std::move(entries)); +} + +////////////////////////////////////////////////////////////////////////// +} diff --git a/src/zip/ZipUtils.cpp b/src/zip/ZipUtils.cpp new file mode 100644 index 0000000..fbb3a21 --- /dev/null +++ b/src/zip/ZipUtils.cpp @@ -0,0 +1,90 @@ +#include "StdAfx.h" +#include "fs/IFilesystem.h" +#include "fs/zip/ZipUtils.h" +#include "fs/zip/ZipPack.h" +#include "fs/WritableFolderPack.h" +#include "fs/CopyStream.h" +#include "fs/zip/ZipWriter.h" +#include "fs/zip/DeflateFileWriter.h" + +namespace fs +{ + +namespace +{ +UnzipResult unzip(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::unique_ref zipPack) +{ + MakeFolderResult makeFolderResult = dstFilesystem.makeFolder(filePath); + if (makeFolderResult.has_error()) + return makeFolderResult.error(); + + for (const EnumerateEntry& ee: zipPack->enumerateRecursively(AbsPath("/"))) + { + AbsPath dstPath = filePath + ee.path; + OUTCOME_TRY(dstFilesystem.makeFolder(ee.isFolder ? dstPath : dstPath.parent())); + if (!ee.isFolder) + { + OUTCOME_TRY(auto source, zipPack->openStreamSource(AbsPath("/") + ee.path)); + OUTCOME_TRY(auto sink, dstFilesystem.openStreamSink(dstPath, Mode::CreateOrOpenAndClear)); + OUTCOME_TRY(copyStream(*sink, *source)); + } + } + + return tl::success(); +} +} + +UnzipResult unzipSource(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::unique_ref source) +{ + OUTCOME_TRY(auto pack, ZipPack::create(std::move(source))); + return unzip(dstFilesystem, filePath, std::move(pack)); +} + + +UnzipResult unzipFile(IFilesystem& dstFilesystem, const AbsPath& filePath, tl::lent_ref filesystem, AbsPath srcFilePath) +{ + OUTCOME_TRY(auto pack, ZipPack::create(std::move(filesystem), std::move(srcFilePath))); + return unzip(dstFilesystem, filePath, std::move(pack)); +} + +ZipResult zipToSink(ISink& sink, tl::lent_ref filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment) +{ + ZipWriter writer(sink, fileDataAlignment); + + OUTCOME_TRY(const bool isFile, filesystem->isFile(path)); + if (isFile) + { + tl::result addResult = writer.addFile(path.str(), [&filesystem, &path, compressionLevel](IStreamSink& sink) + { + return DeflateFileWriter(path, filesystem, compressionLevel)(sink); + }); + if (addResult.has_error()) + return tl::make_error(ErrorCode::SystemError, "Cannot add file '{}' zip: {}", path, addResult.error()); + } + else + { + for (EnumerateEntry ee: filesystem->enumerateRecursively(path)) + { + if (!ee.isFolder) + { + tl::result addResult = writer.addFile(ee.path.get_as(), [&filesystem, &path, &ee, compressionLevel](IStreamSink& sink) + { + return DeflateFileWriter(path + ee.path, filesystem, compressionLevel)(sink); + }); + if (addResult.has_error()) + return {tl::make_error(ErrorCode::SystemError, "Cannot add file '{}' zip: {}", path + ee.path, addResult.error())}; + } + } + } + + writer.finish(); + return tl::success(); +} + +ZipResult zipToFile(IFilesystem& dstFilesystem, const AbsPath& dstFilePath, tl::lent_ref filesystem, const AbsPath& path, uint8_t compressionLevel, size_t fileDataAlignment) +{ + OUTCOME_TRY(const auto sink, dstFilesystem.openSink(dstFilePath, Mode::CreateOrOpenAndClear)); + return zipToSink(*sink, std::move(filesystem), path, compressionLevel, fileDataAlignment); +} + +} diff --git a/src/zip/ZipWriter.cpp b/src/zip/ZipWriter.cpp new file mode 100644 index 0000000..c3ab69d --- /dev/null +++ b/src/zip/ZipWriter.cpp @@ -0,0 +1,229 @@ +#include "StdAfx.h" + +#include "fs/zip/ZipWriter.h" + +#include "fs/CRCStream.h" +#include "fs/IStreamSource.h" +#include "fs/ISink.h" +#include "tl/narrow_cast.h" + +// This writer is implemented pretty close to the https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT standard + +namespace fs +{ +////////////////////////////////////////////////////////////////////////// + +ZipWriter::ZipWriter(ISink& sink, size_t fileDataAlignment) + : m_sink(sink) + , m_fileDataAlignment(fileDataAlignment == 0 ? 1 : fileDataAlignment) +{} + +////////////////////////////////////////////////////////////////////////// + +tl::result ZipWriter::addFile(tl::string name, const DataWriter& dataWriter) +{ + return addFile(std::move(name), + // + {}, + {}, + GeneralBitFlags(), + tl::nullopt, + dataWriter); +} + +tl::result ZipWriter::addFile(tl::string name, tl::optional lastModificationTime, const DataWriter& dataWriter) +{ + return addFile(std::move(name), + // + {}, + {}, + GeneralBitFlags(), + std::move(lastModificationTime), + dataWriter); +} + +////////////////////////////////////////////////////////////////////////// + +tl::result ZipWriter::addFile(tl::string name, + tl::span comment, + tl::span extraField, + GeneralBitFlags generalBitFlags, + tl::optional lastModificationTime, + const DataWriter& dataWriter) +{ + if (name.size() > tl::numeric_limits::max()) + return tl::make_generic_error("Name too long"); + + if (comment.size() > tl::numeric_limits::max()) + return tl::make_generic_error("Comment too long"); + + if (extraField.size() + sizeof(ExtendedInformationExtraField64) > tl::numeric_limits::max()) + return tl::make_generic_error("Extra field too long"); + + ////////////////////////////////////////////////////////////////////////// + // Cannot compute the whole header as we don't know how big the data will be. + // So we compute as much as possible from the header, enough to figure out the padding. + // The we skip to the data section and write it - computing its size at the same time. + // Then we go back to the header section, fill in all the info (since we now know the size, and implictly if it's zip32/64 format) and write the header, name + // etc) NOTE - because of this we always have to write the ExtendedInformationExtraField64 section, even if the final format will not be zip64 + + LocalFileHeader header; + header.extraFieldLength = tl::narrow(extraField.size()) + sizeof(ExtendedInformationExtraField64); + header.generalBitFlag = generalBitFlags.value() | tl::narrow(GeneralBitFlag::LanguageEncodingFlag); + + header.lastModFileDate = 0; + header.lastModFileTime = 0; + if (lastModificationTime != tl::nullopt) + computeMsDosTime(*lastModificationTime, header.lastModFileDate, header.lastModFileTime); + + header.filenameLength = tl::narrow(name.size()); + + const uint64_t padding = computePadding(m_sink.tell(), header); + m_sink.seekRel(padding); + + const uint64_t localHeaderOffset = m_sink.tell(); + + // now skip to the data + m_sink.seekRel(sizeof(LocalFileHeader) + header.filenameLength + header.extraFieldLength); + + const uint64_t startOffset = m_sink.tell(); + + // write the data - this computes the size as well + DataWriterResult dataProviderRes = dataWriter(m_sink); + if (!dataProviderRes) + return dataProviderRes.error(); + + const DataWriterPayload payload = dataProviderRes.value(); + + header.compressionMethod = tl::narrow(payload.compressionMethod); + header.crc32 = payload.uncompressedCRC32; + + const uint64_t endOffset = m_sink.tell(); + const uint64_t compressedSize = endOffset - startOffset; + const uint64_t uncompressedSize = payload.uncompressedSize ? payload.uncompressedSize : compressedSize; + + // write the size fields (in zip64, these are -1) + header.compressedSize = static_cast(-1); + header.uncompressedSize = static_cast(-1); + + // now go back and write the header + m_sink.seekBeg(localHeaderOffset); + m_sink.write({ reinterpret_cast(&header), sizeof(LocalFileHeader) }); + + // the filename + if (!name.empty()) + m_sink.write({ reinterpret_cast(name.data()), name.size() }); + + // extended field - always present! + { + ExtendedInformationExtraField64 extended; + extended.size = static_cast(sizeof(ExtendedInformationExtraField64) - 4); // Substract the tag + size fields. + extended.compressedSize = compressedSize; + extended.originalSize = uncompressedSize; + extended.localHeaderOffset = localHeaderOffset; + m_sink.write({ reinterpret_cast(&extended), sizeof(ExtendedInformationExtraField64) }); + } + + // custom fields + if (!extraField.empty()) + m_sink.write(extraField); + + m_sink.seekBeg(endOffset); + + // store the central directory record + CentralDirectoryRecord record; + record.header = computeCentralDirectoryFileHeader(header, true); + record.compressedSize = compressedSize; + record.uncompressedSize = uncompressedSize; + record.localHeaderOffset = localHeaderOffset; + record.header.localHeaderOffset = static_cast(-1); //zip64 mandates -1 + record.zip64 = true; + record.header.fileCommentLength = tl::narrow(comment.size()); + record.name = std::move(name); + record.comment.assign(comment.begin(), comment.end()); + record.extraField.assign(extraField.begin(), extraField.end()); + m_records.push_back(std::move(record)); + + return tl::success(); +} + +////////////////////////////////////////////////////////////////////////// + +void ZipWriter::finish() +{ + const uint64_t centralDirectoryRecordsOffset = m_sink.tell(); + + for (const CentralDirectoryRecord& record : m_records) + { + TL_ASSERT(record.header.filenameLength == record.name.size()); + TL_ASSERT(record.header.extraFieldLength == record.extraField.size() + (record.zip64 ? sizeof(ExtendedInformationExtraField64) : 0)); + TL_ASSERT(record.header.fileCommentLength == record.comment.size()); + + m_sink.write({ reinterpret_cast(&record.header), sizeof(CentralDirectoryFileHeader) }); + if (!record.name.empty()) + m_sink.write({ reinterpret_cast(record.name.data()), record.name.size() }); + + if (!record.extraField.empty()) + m_sink.write(record.extraField); + + if (record.zip64) + { + ExtendedInformationExtraField64 extended; + extended.size = tl::narrow(sizeof(ExtendedInformationExtraField64) - 4); // Subtract the tag + size fields. + extended.originalSize = record.uncompressedSize; + extended.compressedSize = record.compressedSize; + extended.localHeaderOffset = record.localHeaderOffset; + m_sink.write({ reinterpret_cast(&extended), sizeof(ExtendedInformationExtraField64) }); + } + if (!record.comment.empty()) + m_sink.write(record.comment); + } + const uint64_t sizeOfCentralDirectoryRecords = m_sink.tell() - centralDirectoryRecordsOffset; + + const bool zip64 = centralDirectoryRecordsOffset >= tl::numeric_limits::max() || // + sizeOfCentralDirectoryRecords >= tl::numeric_limits::max() || // + m_records.size() >= tl::numeric_limits::max(); + + if (zip64) + { + const uint64_t endOfCentralDirectoryRecord64Offset = m_sink.tell(); + + { + EndOfCentralDirectoryRecord64 header; + header.endOfCentralDirectorySize = sizeof(EndOfCentralDirectoryRecord64) - 12; + header.centralDirectoryRecordCountOnThisDisk = m_records.size(); + header.centralDirectoryRecordCount = m_records.size(); + header.centralDirectorySize = sizeOfCentralDirectoryRecords; + header.centralDirectoryOffset = centralDirectoryRecordsOffset; + m_sink.write({ reinterpret_cast(&header), sizeof(EndOfCentralDirectoryRecord64) }); + } + { + EndOfCentralDirectoryLocator64 header; + header.endOfCentralDirectoryRecordOffset = endOfCentralDirectoryRecord64Offset; + header.diskCount = 1; + m_sink.write({ reinterpret_cast(&header), sizeof(EndOfCentralDirectoryLocator64) }); + } + } + + EndOfCentralDirectoryRecord header; + header.centralDirectoryRecordCountOnThisDisk = zip64 ? static_cast(-1) : tl::narrow(m_records.size()); + header.centralDirectoryRecordCount = zip64 ? static_cast(-1) : tl::narrow(m_records.size()); + header.centralDirectorySize = zip64 ? static_cast(-1) : tl::narrow(sizeOfCentralDirectoryRecords); + header.centralDirectoryOffset = zip64 ? static_cast(-1) : tl::narrow(centralDirectoryRecordsOffset); + m_sink.write({ reinterpret_cast(&header), sizeof(EndOfCentralDirectoryRecord) }); +} + +////////////////////////////////////////////////////////////////////////// + +inline uint64_t ZipWriter::computePadding(uint64_t offset, const LocalFileHeader& header) const +{ + const uint64_t dataOffset = offset + sizeof(LocalFileHeader) + header.filenameLength + header.extraFieldLength; + const uint64_t m = dataOffset % m_fileDataAlignment; + if (m == 0) + return 0; + + return m_fileDataAlignment - m; +} + +////////////////////////////////////////////////////////////////////////// +}