diff --git a/.appveyor.yml b/.appveyor.yml index 1e143afa7..f76b3b321 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -24,66 +24,25 @@ clone_depth: 50 #same as travis, see https://www.appveyor.com/blog/2014/06/04 image: Visual Studio 2017 cache: - - c:\openssl-release - - c:\protobuf-release - - c:\zlib-release -# TODO: set dependency on ps skript file (in ./ci) / "-> appveyor.yml" maybe not ideal -# that way when we update specific cached tools there (like protobuf or zlib), cache will be newly created automatically -# https://www.appveyor.com/docs/build-cache/#cleaning-up-cache - + - c:\Tools\vcpkg\installed environment: - openssl_ver: 1.0.2p - protobuf_ver: 3.6.1 - zlib_ver: 1.2.11 - matrix: - target_arch: win64 qt_ver: 5.9\msvc2017_64 cmake_generator: Visual Studio 15 2017 Win64 cmake_toolset: v141,host=x64 - vc_arch: amd64 + vcpkg_arch: x64 - target_arch: win32 qt_ver: 5.9\msvc2015 # Qt doesn't provide a msvc2017_32 cmake_generator: Visual Studio 15 2017 cmake_toolset: v141 - vc_arch: amd64_x86 - + vcpkg_arch: x86 install: - - ps: | - if (Test-Path c:\openssl-release) { - echo "using openssl from cache" - } else { - if ($env:target_arch -eq "win64") { # 64bit filename - # echo "downloading 64bit version of openssl" - Invoke-WebRequest "https://indy.fulgan.com/SSL/openssl-$env:openssl_ver-x64_86-win64.zip" -OutFile c:\openssl-$env:openssl_ver.zip - } else { # 32bit filename - # echo "downloading 32bit version of openssl" - Invoke-WebRequest "https://indy.fulgan.com/SSL/openssl-$env:openssl_ver-i386-win32.zip" -OutFile c:\openssl-$env:openssl_ver.zip - } - Expand-Archive -Path c:\openssl-$env:openssl_ver.zip -DestinationPath c:\openssl-release - Set-Location -Path C:\openssl-release - } - if (Test-Path c:\protobuf-release) { - echo "using protobuf from cache" - } else { - Invoke-WebRequest "https://github.com/protocolbuffers/protobuf/releases/download/v$env:protobuf_ver/protobuf-cpp-$env:protobuf_ver.zip" -OutFile c:\protobuf-cpp-$env:protobuf_ver.zip - Expand-Archive -Path c:\protobuf-cpp-$env:protobuf_ver.zip -DestinationPath c:\ - Set-Location -Path C:\protobuf-$env:protobuf_ver\cmake - cmake . -G "$env:cmake_generator" -T "$env:cmake_toolset" -Dprotobuf_BUILD_TESTS=0 -Dprotobuf_MSVC_STATIC_RUNTIME=0 -DCMAKE_INSTALL_PREFIX=c:/protobuf-release - msbuild INSTALL.vcxproj /p:Configuration=Release - } - if (Test-Path c:\zlib-release) { - echo "using zlib from cache" - } else { - Invoke-WebRequest "https://github.com/madler/zlib/archive/v$env:zlib_ver.zip" -OutFile c:\zlib-$env:zlib_ver.zip - Expand-Archive -Path c:\zlib-$env:zlib_ver.zip -DestinationPath c:\ - Set-Location -Path C:\zlib-$env:zlib_ver - cmake . -G "$env:cmake_generator" -T "$env:cmake_toolset" -DCMAKE_INSTALL_PREFIX=c:/zlib-release - msbuild INSTALL.vcxproj /p:Configuration=Release - } + - vcpkg remove --outdated --recurse + - vcpkg install openssl protobuf liblzma zlib --triplet %vcpkg_arch%-windows services: - mysql @@ -92,13 +51,10 @@ build_script: - ps: | New-Item -ItemType directory -Path $env:APPVEYOR_BUILD_FOLDER\build Set-Location -Path $env:APPVEYOR_BUILD_FOLDER\build - $zlibdir = "c:\zlib-release" - $openssldir = "C:\openssl-release" - $protodir = "c:\protobuf-release" - $protoc = "c:\protobuf-release\bin\protoc.exe" + $vcpkgbindir = "C:\Tools\vcpkg\installed\$env:vcpkg_arch-windows\bin" $mysqldll = "c:\Program Files\MySQL\MySQL Server 5.7\lib\libmysql.dll" cmake --version - cmake .. -G "$env:cmake_generator" -T "$env:cmake_toolset" "-DCMAKE_PREFIX_PATH=c:/Qt/$env:qt_ver;$protodir;$zlibdir;$openssldir" "-DWITH_SERVER=1" "-DPROTOBUF_PROTOC_EXECUTABLE=$protoc" "-DMYSQLCLIENT_LIBRARIES=$mysqldll" + cmake .. -G "$env:cmake_generator" -T "$env:cmake_toolset" "-DCMAKE_TOOLCHAIN_FILE=c:/tools/vcpkg/scripts/buildsystems/vcpkg.cmake" "-DCMAKE_PREFIX_PATH=c:/Qt/$env:qt_ver;$vcpkgbindir" "-DWITH_SERVER=1" "-DMYSQLCLIENT_LIBRARIES=$mysqldll" - msbuild PACKAGE.vcxproj /p:Configuration=Release - ps: | $exe = dir -name *.exe @@ -110,7 +66,7 @@ build_script: (New-Object PSObject | Add-Member -PassThru NoteProperty bin $new_name | Add-Member -PassThru NoteProperty cmake $cmake_name | Add-Member -PassThru NoteProperty commit $env:APPVEYOR_REPO_COMMIT) | ConvertTo-JSON | Out-File -FilePath "latest-$env:target_arch" -Encoding ASCII Push-AppveyorArtifact "latest-$env:target_arch" $version = $matches['content'] - + test: off diff --git a/.ci/Fedora29/Dockerfile b/.ci/Fedora29/Dockerfile index 77ee4e9f3..7b0e9cf68 100644 --- a/.ci/Fedora29/Dockerfile +++ b/.ci/Fedora29/Dockerfile @@ -15,4 +15,5 @@ RUN dnf install -y \ sqlite-devel \ wget \ zlib-devel \ + xz-devel \ && dnf clean all diff --git a/.ci/UbuntuBionic/Dockerfile b/.ci/UbuntuBionic/Dockerfile index 9c41fb638..96e120a8d 100644 --- a/.ci/UbuntuBionic/Dockerfile +++ b/.ci/UbuntuBionic/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ ccache \ cmake \ + liblzma-dev \ libprotobuf-dev \ libqt5multimedia5-plugins \ libqt5svg5-dev \ diff --git a/.travis.yml b/.travis.yml index 239e98ea0..b20c35a97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ matrix: packages: - libprotobuf-dev - protobuf-compiler + - liblzma-dev - qt5-default - qttools5-dev - qttools5-dev-tools @@ -87,9 +88,7 @@ matrix: env: HOMEBREW_NO_AUTO_UPDATE=1 cache: ccache before_install: - - brew install ccache - - brew install protobuf - - brew install qt + - brew install ccache protobuf qt xz script: bash ./.ci/travis-compile.sh --server --install --debug - name: macOS (Release) @@ -99,9 +98,7 @@ matrix: env: HOMEBREW_NO_AUTO_UPDATE=1 cache: ccache before_install: - - brew install ccache - - brew install protobuf - - brew install qt + - brew install ccache protobuf qt xz script: bash ./.ci/travis-compile.sh --server --package "$TRAVIS_OS_NAME" --release diff --git a/CMakeLists.txt b/CMakeLists.txt index d5f75540e..f55835061 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,7 +99,8 @@ option(WARNING_AS_ERROR "Treat warnings as errors in debug builds" ON) IF(MSVC) # Visual Studio: # Maximum optimization - set(CMAKE_CXX_FLAGS_RELEASE "/Ox /MD") + # Disable warning C4251 + set(CMAKE_CXX_FLAGS_RELEASE "/Ox /MD /wd4251") # Generate complete debugging information #set(CMAKE_CXX_FLAGS_DEBUG "/Zi") ELSEIF (CMAKE_COMPILER_IS_GNUCXX) diff --git a/README.md b/README.md index cfcf41fdb..82f11d28a 100644 --- a/README.md +++ b/README.md @@ -87,8 +87,9 @@ Dependencies: *(for minimum requirements search our [CMake file](https://github. - [protobuf](https://github.com/google/protobuf) - [CMake](https://www.cmake.org/) -Oracle can optionally use zlib to load zipped files: +Oracle can optionally use zlib and xz to load compressed files: - [zlib](https://www.zlib.net/) +- [xz](https://tukaani.org/xz/) To compile: diff --git a/clangify.sh b/clangify.sh index ca5b4ace9..ce81a1140 100755 --- a/clangify.sh +++ b/clangify.sh @@ -15,7 +15,8 @@ include=("common" \ exclude=("cockatrice/src/qt-json" \ "servatrice/src/smtp" \ "common/sfmt" \ -"oracle/src/zip") +"oracle/src/zip" \ +"oracle/src/lzma") exts=("cpp" "h") cf_cmd="clang-format" branch="origin/master" diff --git a/cmake/FindWin32SslRuntime.cmake b/cmake/FindWin32SslRuntime.cmake index 9cca2f7bc..79ba688d4 100644 --- a/cmake/FindWin32SslRuntime.cmake +++ b/cmake/FindWin32SslRuntime.cmake @@ -2,7 +2,7 @@ # will be needed by Qt in order to access https urls. if (WIN32) - # Get standard installation paths for OpenSSL under Windows + # Get standard installation paths for OpenSSL under Windows # http://www.slproweb.com/products/Win32OpenSSL.html @@ -15,6 +15,7 @@ if (WIN32) ) file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles) set(_OPENSSL_ROOT_PATHS + "C:/Tools/vcpkg/installed/x64-windows/bin" "${_programfiles}/OpenSSL-Win64" "C:/OpenSSL-Win64/" ) @@ -28,6 +29,7 @@ if (WIN32) ) file(TO_CMAKE_PATH "$ENV{PROGRAMFILES}" _programfiles) set(_OPENSSL_ROOT_PATHS + "C:/Tools/vcpkg/installed/x86-windows/bin" "${_programfiles}/OpenSSL" "${_programfiles}/OpenSSL-Win32" "C:/OpenSSL/" diff --git a/cockatrice/CMakeLists.txt b/cockatrice/CMakeLists.txt index 064e17701..3c62074d9 100644 --- a/cockatrice/CMakeLists.txt +++ b/cockatrice/CMakeLists.txt @@ -254,6 +254,8 @@ if(WIN32) set(plugin_dest_dir Plugins) set(qtconf_dest_dir .) + install(DIRECTORY "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/${CMAKE_BUILD_TYPE}/" DESTINATION ./ FILES_MATCHING PATTERN "*.dll") + # qt5 plugins: audio, iconengines, imageformats, platforms, printsupport install(DIRECTORY "${QT_PLUGINS_DIR}/" DESTINATION ${plugin_dest_dir} COMPONENT Runtime FILES_MATCHING REGEX "(audio|iconengines|imageformats|platforms|printsupport)/.*[^d]\\.dll") diff --git a/cockatrice/src/carddatabase.cpp b/cockatrice/src/carddatabase.cpp index 9b2137791..f4995db5f 100644 --- a/cockatrice/src/carddatabase.cpp +++ b/cockatrice/src/carddatabase.cpp @@ -224,13 +224,14 @@ CardInfo::CardInfo(const QString &_name, const SetList &_sets, const QStringMap &_customPicURLs, MuidMap _muIds, + QStringMap _uuIds, QStringMap _collectorNumbers, QStringMap _rarities) : name(_name), isToken(_isToken), sets(_sets), manacost(_manacost), cmc(_cmc), cardtype(_cardtype), powtough(_powtough), text(_text), colors(_colors), relatedCards(_relatedCards), reverseRelatedCards(_reverseRelatedCards), setsNames(), upsideDownArt(_upsideDownArt), loyalty(_loyalty), - customPicURLs(_customPicURLs), muIds(std::move(_muIds)), collectorNumbers(std::move(_collectorNumbers)), - rarities(std::move(_rarities)), cipt(_cipt), tableRow(_tableRow) + customPicURLs(_customPicURLs), muIds(std::move(_muIds)), uuIds(std::move(_uuIds)), + collectorNumbers(std::move(_collectorNumbers)), rarities(std::move(_rarities)), cipt(_cipt), tableRow(_tableRow) { pixmapCacheKey = QLatin1String("card_") + name; simpleName = CardInfo::simplifyName(name); @@ -260,12 +261,13 @@ CardInfoPtr CardInfo::newInstance(const QString &_name, const SetList &_sets, const QStringMap &_customPicURLs, MuidMap _muIds, + QStringMap _uuIds, QStringMap _collectorNumbers, QStringMap _rarities) { CardInfoPtr ptr(new CardInfo(_name, _isToken, _manacost, _cmc, _cardtype, _powtough, _text, _colors, _relatedCards, _reverseRelatedCards, _upsideDownArt, _loyalty, _cipt, _tableRow, _sets, - _customPicURLs, std::move(_muIds), std::move(_collectorNumbers), + _customPicURLs, std::move(_muIds), std::move(_uuIds), std::move(_collectorNumbers), std::move(_rarities))); ptr->setSmartPointer(ptr); @@ -440,6 +442,7 @@ void CardDatabase::addCard(CardInfoPtr card) QString setName = set->getCorrectedShortName(); sameCard->setSet(set); sameCard->setMuId(setName, card->getMuId(setName)); + sameCard->setUuId(setName, card->getUuId(setName)); sameCard->setRarity(setName, card->getRarity(setName)); sameCard->setSetNumber(setName, card->getCollectorNumber(setName)); } diff --git a/cockatrice/src/carddatabase.h b/cockatrice/src/carddatabase.h index c58c1ad83..8aa89e0b2 100644 --- a/cockatrice/src/carddatabase.h +++ b/cockatrice/src/carddatabase.h @@ -149,6 +149,7 @@ private: QString loyalty; QStringMap customPicURLs; MuidMap muIds; + QStringMap uuIds; QStringMap collectorNumbers; QStringMap rarities; bool cipt; @@ -172,7 +173,8 @@ public: int _tableRow = 0, const SetList &_sets = SetList(), const QStringMap &_customPicURLs = QStringMap(), - MuidMap muids = MuidMap(), + MuidMap _muids = MuidMap(), + QStringMap _uuIds = QStringMap(), QStringMap _collectorNumbers = QStringMap(), QStringMap _rarities = QStringMap()); ~CardInfo() override; @@ -193,7 +195,8 @@ public: int _tableRow = 0, const SetList &_sets = SetList(), const QStringMap &_customPicURLs = QStringMap(), - MuidMap muids = MuidMap(), + MuidMap _muids = MuidMap(), + QStringMap _uuIds = QStringMap(), QStringMap _collectorNumbers = QStringMap(), QStringMap _rarities = QStringMap()); @@ -310,6 +313,10 @@ public: { return muIds.value(set); } + QString getUuId(const QString &set) const + { + return uuIds.value(set); + } QString getCollectorNumber(const QString &set) const { return collectorNumbers.value(set); @@ -344,6 +351,10 @@ public: { muIds.insert(_set, _muId); } + void setUuId(const QString &_set, const QString &_uuId) + { + uuIds.insert(_set, _uuId); + } void setSetNumber(const QString &_set, const QString &_setNumber) { collectorNumbers.insert(_set, _setNumber); diff --git a/cockatrice/src/carddbparser/cockatricexml3.cpp b/cockatrice/src/carddbparser/cockatricexml3.cpp index d98202ac6..6a6300cf4 100644 --- a/cockatrice/src/carddbparser/cockatricexml3.cpp +++ b/cockatrice/src/carddbparser/cockatricexml3.cpp @@ -133,7 +133,7 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml) QList relatedCards, reverseRelatedCards; QStringMap customPicURLs; MuidMap muids; - QStringMap collectorNumbers, rarities; + QStringMap uuids, collectorNumbers, rarities; SetList sets; int tableRow = 0; bool cipt = false; @@ -164,6 +164,10 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml) muids[setName] = attrs.value("muId").toString().toInt(); } + if (attrs.hasAttribute("muId")) { + uuids[setName] = attrs.value("uuId").toString(); + } + if (attrs.hasAttribute("picURL")) { customPicURLs[setName] = attrs.value("picURL").toString(); } @@ -232,7 +236,7 @@ void CockatriceXml3Parser::loadCardsFromXml(QXmlStreamReader &xml) CardInfoPtr newCard = CardInfo::newInstance( name, isToken, manacost, cmc, type, pt, text, colors, relatedCards, reverseRelatedCards, upsideDown, - loyalty, cipt, tableRow, sets, customPicURLs, muids, collectorNumbers, rarities); + loyalty, cipt, tableRow, sets, customPicURLs, muids, uuids, collectorNumbers, rarities); emit addCard(newCard); } } @@ -274,6 +278,7 @@ static QXmlStreamWriter &operator<<(QXmlStreamWriter &xml, const CardInfoPtr &in tmpSet = sets[i]->getShortName(); xml.writeAttribute("rarity", info->getRarity(tmpSet)); xml.writeAttribute("muId", QString::number(info->getMuId(tmpSet))); + xml.writeAttribute("uuId", info->getUuId(tmpSet)); tmpString = info->getCollectorNumber(tmpSet); if (!tmpString.isEmpty()) { diff --git a/cockatrice/src/decklistmodel.cpp b/cockatrice/src/decklistmodel.cpp index e8b4b95ff..8b6e7bf54 100644 --- a/cockatrice/src/decklistmodel.cpp +++ b/cockatrice/src/decklistmodel.cpp @@ -324,7 +324,7 @@ QModelIndex DeckListModel::addCard(const QString &cardName, const QString &zoneN // and default values for all fields info = CardInfo::newInstance(cardName, false, nullptr, nullptr, "unknown", nullptr, nullptr, QStringList(), QList(), QList(), false, 0, false, 0, - SetList(), QStringMap(), MuidMap(), QStringMap(), QStringMap()); + SetList(), QStringMap(), MuidMap(), QStringMap(), QStringMap(), QStringMap()); } else { return {}; } diff --git a/cockatrice/src/pictureloader.cpp b/cockatrice/src/pictureloader.cpp index 341ad970d..7d3e32ff6 100644 --- a/cockatrice/src/pictureloader.cpp +++ b/cockatrice/src/pictureloader.cpp @@ -267,6 +267,7 @@ QString PictureToLoad::transformUrl(QString urlTemplate) const if (set) { transformMap["!cardid!"] = QString::number(card->getMuId(set->getShortName())); + transformMap["!uuid!"] = card->getUuId(set->getShortName()); transformMap["!collectornumber!"] = card->getCollectorNumber(set->getShortName()); transformMap["!setcode!"] = set->getShortName(); transformMap["!setcode_lower!"] = set->getShortName().toLower(); @@ -274,6 +275,7 @@ QString PictureToLoad::transformUrl(QString urlTemplate) const transformMap["!setname_lower!"] = set->getLongName().toLower(); } else { transformMap["!cardid!"] = QString(); + transformMap["!uuid!"] = QString(); transformMap["!collectornumber!"] = QString(); transformMap["!setcode!"] = QString(); transformMap["!setcode_lower!"] = QString(); diff --git a/doc/cards.xsd b/doc/cards.xsd index d3721e9bb..89ae6eb95 100644 --- a/doc/cards.xsd +++ b/doc/cards.xsd @@ -39,6 +39,7 @@ + diff --git a/oracle/CMakeLists.txt b/oracle/CMakeLists.txt index 5b3567fcc..ec7b5d171 100644 --- a/oracle/CMakeLists.txt +++ b/oracle/CMakeLists.txt @@ -85,18 +85,33 @@ IF(ZLIB_FOUND) src/zip/unzip.cpp src/zip/zipglobal.cpp ) - ELSE() MESSAGE(STATUS "Oracle: zlib not found; ZIP support disabled") ENDIF() +# LibLZMA is required to support xz files +FIND_PACKAGE(LibLZMA) +IF(LIBLZMA_FOUND) + INCLUDE_DIRECTORIES(${LIBLZMA_INCLUDE_DIRS}) + ADD_DEFINITIONS("-DHAS_LZMA") + + set(oracle_SOURCES ${oracle_SOURCES} + src/lzma/decompress.cpp + ) +ELSE() + MESSAGE(STATUS "Oracle: LibLZMA not found; xz support disabled") +ENDIF() + # Build oracle binary and link it ADD_EXECUTABLE(oracle WIN32 MACOSX_BUNDLE ${oracle_SOURCES} ${oracle_QM} ${oracle_RESOURCES_RCC} ${oracle_MOC_SRCS}) +TARGET_LINK_LIBRARIES(oracle ${ORACLE_QT_MODULES}) IF(ZLIB_FOUND) - TARGET_LINK_LIBRARIES(oracle ${ORACLE_QT_MODULES} ${ZLIB_LIBRARIES}) -ELSE() - TARGET_LINK_LIBRARIES(oracle ${ORACLE_QT_MODULES}) + TARGET_LINK_LIBRARIES(oracle ${ZLIB_LIBRARIES}) +ENDIF() + +IF(LIBLZMA_FOUND) + TARGET_LINK_LIBRARIES(oracle ${LIBLZMA_LIBRARIES}) ENDIF() if(UNIX) @@ -163,14 +178,8 @@ IF(WIN32) set(plugin_dest_dir Plugins) set(qtconf_dest_dir .) list(APPEND libSearchDirs ${QT_LIBRARY_DIR}) - IF(ZLIB_FOUND) - # look for dll in the bin/ directory (gnuwin32 package) - get_filename_component(ZLIB_DLL_DIR "${ZLIB_INCLUDE_DIRS}/../bin/" REALPATH) - list(APPEND libSearchDirs ${ZLIB_DLL_DIR}) - # look for dll in the lib/ directory (nuget package) - get_filename_component(ZLIB_DLL_DIR "${ZLIB_LIBRARY}" DIRECTORY) - list(APPEND libSearchDirs ${ZLIB_DLL_DIR}) - ENDIF() + + install(DIRECTORY "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/${CMAKE_BUILD_TYPE}/" DESTINATION ./ FILES_MATCHING PATTERN "*.dll") # qt5 plugins: iconengines, platforms diff --git a/oracle/src/lzma/decompress.cpp b/oracle/src/lzma/decompress.cpp new file mode 100644 index 000000000..718cde207 --- /dev/null +++ b/oracle/src/lzma/decompress.cpp @@ -0,0 +1,250 @@ +/* + * Simple routing to extract a single file from a xz archive + * Heavily based from doc/examples/02_decompress.c obtained from + * the official xz git repository: git.tukaani.org/xz.git + * The license from the original file header follows + * + * Author: Lasse Collin + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + + +#include +#include + +#include "decompress.h" + +XzDecompressor::XzDecompressor(QObject *parent) + : QObject(parent) +{ + +} + +bool XzDecompressor::decompress(QBuffer *in, QBuffer *out) +{ + lzma_stream strm = LZMA_STREAM_INIT; + bool success; + + if (!init_decoder(&strm)) { + return false; + } + + success = internal_decompress(&strm, in, out); + + // Free the memory allocated for the decoder. This only needs to be + // done after the last file. + lzma_end(&strm); + + return success; +} + +bool XzDecompressor::init_decoder(lzma_stream *strm) +{ + // Initialize a .xz decoder. The decoder supports a memory usage limit + // and a set of flags. + // + // The memory usage of the decompressor depends on the settings used + // to compress a .xz file. It can vary from less than a megabyte to + // a few gigabytes, but in practice (at least for now) it rarely + // exceeds 65 MiB because that's how much memory is required to + // decompress files created with "xz -9". Settings requiring more + // memory take extra effort to use and don't (at least for now) + // provide significantly better compression in most cases. + // + // Memory usage limit is useful if it is important that the + // decompressor won't consume gigabytes of memory. The need + // for limiting depends on the application. In this example, + // no memory usage limiting is used. This is done by setting + // the limit to UINT64_MAX. + // + // The .xz format allows concatenating compressed files as is: + // + // echo foo | xz > foobar.xz + // echo bar | xz >> foobar.xz + // + // When decompressing normal standalone .xz files, LZMA_CONCATENATED + // should always be used to support decompression of concatenated + // .xz files. If LZMA_CONCATENATED isn't used, the decoder will stop + // after the first .xz stream. This can be useful when .xz data has + // been embedded inside another file format. + // + // Flags other than LZMA_CONCATENATED are supported too, and can + // be combined with bitwise-or. See lzma/container.h + // (src/liblzma/api/lzma/container.h in the source package or e.g. + // /usr/include/lzma/container.h depending on the install prefix) + // for details. + lzma_ret ret = lzma_stream_decoder( + strm, UINT64_MAX, LZMA_CONCATENATED); + + // Return successfully if the initialization went fine. + if (ret == LZMA_OK) + return true; + + // Something went wrong. The possible errors are documented in + // lzma/container.h (src/liblzma/api/lzma/container.h in the source + // package or e.g. /usr/include/lzma/container.h depending on the + // install prefix). + // + // Note that LZMA_MEMLIMIT_ERROR is never possible here. If you + // specify a very tiny limit, the error will be delayed until + // the first headers have been parsed by a call to lzma_code(). + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = "Memory allocation failed"; + break; + + case LZMA_OPTIONS_ERROR: + msg = "Unsupported decompressor flags"; + break; + + default: + // This is most likely LZMA_PROG_ERROR indicating a bug in + // this program or in liblzma. It is inconvenient to have a + // separate error message for errors that should be impossible + // to occur, but knowing the error code is important for + // debugging. That's why it is good to print the error code + // at least when there is no good error message to show. + msg = "Unknown error, possibly a bug"; + break; + } + + qDebug() << "Error initializing the decoder:" << msg << "(error code " << ret << ")"; + return false; +} + + +bool XzDecompressor::internal_decompress(lzma_stream *strm, QBuffer *in, QBuffer *out) +{ + // When LZMA_CONCATENATED flag was used when initializing the decoder, + // we need to tell lzma_code() when there will be no more input. + // This is done by setting action to LZMA_FINISH instead of LZMA_RUN + // in the same way as it is done when encoding. + // + // When LZMA_CONCATENATED isn't used, there is no need to use + // LZMA_FINISH to tell when all the input has been read, but it + // is still OK to use it if you want. When LZMA_CONCATENATED isn't + // used, the decoder will stop after the first .xz stream. In that + // case some unused data may be left in strm->next_in. + lzma_action action = LZMA_RUN; + + uint8_t inbuf[BUFSIZ]; + uint8_t outbuf[BUFSIZ]; + qint64 bytesAvailable; + + strm->next_in = NULL; + strm->avail_in = 0; + strm->next_out = outbuf; + strm->avail_out = sizeof(outbuf); + while (true) { + if (strm->avail_in == 0) { + strm->next_in = inbuf; + bytesAvailable = in->bytesAvailable(); + if(bytesAvailable == 0) { + // Once the end of the input file has been reached, + // we need to tell lzma_code() that no more input + // will be coming. As said before, this isn't required + // if the LZMA_CONCATENATED flag isn't used when + // initializing the decoder. + action = LZMA_FINISH; + } else if(bytesAvailable >= BUFSIZ) { + in->read((char*) inbuf, BUFSIZ); + strm->avail_in = BUFSIZ; + } else { + in->read((char*) inbuf, bytesAvailable); + strm->avail_in = bytesAvailable; + } + } + + lzma_ret ret = lzma_code(strm, action); + + if (strm->avail_out == 0 || ret == LZMA_STREAM_END) { + qint64 write_size = sizeof(outbuf) - strm->avail_out; + + if (out->write((char *) outbuf, write_size) != write_size) { + qDebug() << "Write error"; + return false; + } + + strm->next_out = outbuf; + strm->avail_out = sizeof(outbuf); + } + + if (ret != LZMA_OK) { + // Once everything has been decoded successfully, the + // return value of lzma_code() will be LZMA_STREAM_END. + // + // It is important to check for LZMA_STREAM_END. Do not + // assume that getting ret != LZMA_OK would mean that + // everything has gone well or that when you aren't + // getting more output it must have successfully + // decoded everything. + if (ret == LZMA_STREAM_END) + return true; + + // It's not LZMA_OK nor LZMA_STREAM_END, + // so it must be an error code. See lzma/base.h + // (src/liblzma/api/lzma/base.h in the source package + // or e.g. /usr/include/lzma/base.h depending on the + // install prefix) for the list and documentation of + // possible values. Many values listen in lzma_ret + // enumeration aren't possible in this example, but + // can be made possible by enabling memory usage limit + // or adding flags to the decoder initialization. + const char *msg; + switch (ret) { + case LZMA_MEM_ERROR: + msg = "Memory allocation failed"; + break; + + case LZMA_FORMAT_ERROR: + // .xz magic bytes weren't found. + msg = "The input is not in the .xz format"; + break; + + case LZMA_OPTIONS_ERROR: + // For example, the headers specify a filter + // that isn't supported by this liblzma + // version (or it hasn't been enabled when + // building liblzma, but no-one sane does + // that unless building liblzma for an + // embedded system). Upgrading to a newer + // liblzma might help. + // + // Note that it is unlikely that the file has + // accidentally became corrupt if you get this + // error. The integrity of the .xz headers is + // always verified with a CRC32, so + // unintentionally corrupt files can be + // distinguished from unsupported files. + msg = "Unsupported compression options"; + break; + + case LZMA_DATA_ERROR: + msg = "Compressed file is corrupt"; + break; + + case LZMA_BUF_ERROR: + // Typically this error means that a valid + // file has got truncated, but it might also + // be a damaged part in the file that makes + // the decoder think the file is truncated. + // If you prefer, you can use the same error + // message for this as for LZMA_DATA_ERROR. + msg = "Compressed file is truncated or " + "otherwise corrupt"; + break; + + default: + // This is most likely LZMA_PROG_ERROR. + msg = "Unknown error, possibly a bug"; + break; + } + + qDebug() << "Decoder error:" << msg << "(error code " << ret << ")"; + return false; + } + } +} + diff --git a/oracle/src/lzma/decompress.h b/oracle/src/lzma/decompress.h new file mode 100644 index 000000000..f0e315f8b --- /dev/null +++ b/oracle/src/lzma/decompress.h @@ -0,0 +1,19 @@ +#ifndef XZ_DECOMPRESS_H +#define XZ_DECOMPRESS_H + +#include +#include + +class XzDecompressor : public QObject +{ + Q_OBJECT +public: + XzDecompressor(QObject *parent = 0); + ~XzDecompressor() { }; + bool decompress(QBuffer *in, QBuffer *out); +private: + bool init_decoder(lzma_stream *strm); + bool internal_decompress(lzma_stream *strm, QBuffer *in, QBuffer *out); +}; + +#endif diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 90c908f91..4ac937cc3 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -19,7 +19,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) setsMap = QtJson::Json::parse(QString(data), ok).toMap(); if (!ok) { qDebug() << "error: QtJson::Json::parse()"; - return 0; + return false; } QListIterator it(setsMap.values()); @@ -33,7 +33,7 @@ bool OracleImporter::readSetsFromByteArray(const QByteArray &data) while (it.hasNext()) { map = it.next().toMap(); - edition = map.value("code").toString(); + edition = map.value("code").toString().toUpper(); editionLong = map.value("name").toString(); editionCards = map.value("cards"); setType = map.value("type").toString(); @@ -57,6 +57,7 @@ CardInfoPtr OracleImporter::addCard(const QString &setName, QString cardName, bool isToken, int cardId, + QString &cardUuId, QString &setNumber, QString &cardCost, QString &cmc, @@ -124,30 +125,13 @@ CardInfoPtr OracleImporter::addCard(const QString &setName, cards.insert(cardName, card); } card->setMuId(setName, cardId); + card->setUuId(setName, cardUuId); card->setSetNumber(setName, setNumber); card->setRarity(setName, rarity); return card; } -void OracleImporter::extractColors(const QStringList &in, QStringList &out) -{ - foreach (QString c, in) { - if (c == "White") - out << "W"; - else if (c == "Blue") - out << "U"; - else if (c == "Black") - out << "B"; - else if (c == "Red") - out << "R"; - else if (c == "Green") - out << "G"; - else - qDebug() << "error: unknown color:" << c; - } -} - int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) { int cards = 0; @@ -164,6 +148,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) QList relatedCards; QList reverseRelatedCards; // dummy int cardId; + QString cardUuId; QString setNumber; QString rarity; QString cardLoyalty; @@ -173,15 +158,20 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) while (it.hasNext()) { map = it.next().toMap(); + /* Currently used layouts are: + * augment, double_faced_token, flip, host, leveler, meld, normal, planar, + * saga, scheme, split, token, transform, vanguard + */ QString layout = map.value("layout").toString(); // don't import tokens from the json file if (layout == "token") continue; - if (layout == "split" || layout == "aftermath") { + // Aftermath card layout seems to have been integrated in "split" + if (layout == "split") { // Enqueue split card for later handling - cardId = map.contains("multiverseid") ? map.value("multiverseid").toInt() : 0; + cardId = map.contains("multiverseId") ? map.value("multiverseId").toInt() : 0; if (cardId) splitCards.insertMulti(cardId, map); continue; @@ -190,16 +180,18 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) // normal cards handling cardName = map.contains("name") ? map.value("name").toString() : QString(""); cardCost = map.contains("manaCost") ? map.value("manaCost").toString() : QString(""); - cmc = map.contains("cmc") ? map.value("cmc").toString() : QString("0"); + cmc = map.contains("convertedManaCost") ? map.value("convertedManaCost").toString() : QString("0"); cardType = map.contains("type") ? map.value("type").toString() : QString(""); cardPT = map.contains("power") || map.contains("toughness") ? map.value("power").toString() + QString('/') + map.value("toughness").toString() : QString(""); cardText = map.contains("text") ? map.value("text").toString() : QString(""); - cardId = map.contains("multiverseid") ? map.value("multiverseid").toInt() : 0; + cardId = map.contains("multiverseId") ? map.value("multiverseId").toInt() : 0; + cardUuId = map.contains("uuid") ? map.value("uuid").toString() : QString(""); setNumber = map.contains("number") ? map.value("number").toString() : QString(""); rarity = map.contains("rarity") ? map.value("rarity").toString() : QString(""); cardLoyalty = map.contains("loyalty") ? map.value("loyalty").toString() : QString(""); + colors = map.contains("colors") ? map.value("colors").toStringList() : QStringList(); relatedCards = QList(); if (map.contains("names")) foreach (const QString &name, map.value("names").toStringList()) { @@ -214,11 +206,8 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) upsideDown = false; } - colors.clear(); - extractColors(map.value("colors").toStringList(), colors); - CardInfoPtr card = - addCard(set->getShortName(), cardName, false, cardId, setNumber, cardCost, cmc, cardType, cardPT, + addCard(set->getShortName(), cardName, false, cardId, cardUuId, setNumber, cardCost, cmc, cardType, cardPT, cardLoyalty, cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity); if (!set->contains(card)) { @@ -250,6 +239,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) cardType = ""; cardPT = ""; cardText = ""; + cardUuId = ""; setNumber = ""; rarity = ""; cardLoyalty = ""; @@ -269,10 +259,10 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) cardCost += prefix; cardCost += map.value("manaCost").toString(); } - if (map.contains("cmc")) { + if (map.contains("convertedManaCost")) { if (!cmc.isEmpty()) cmc += prefix; - cmc += map.value("cmc").toString(); + cmc += map.value("convertedManaCost").toString(); } if (map.contains("type")) { if (!cardType.isEmpty()) @@ -289,6 +279,10 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) cardText += prefix2; cardText += map.value("text").toString(); } + if (map.contains("uuid")) { + if (cardUuId.isEmpty()) + cardUuId = map.value("uuid").toString(); + } if (map.contains("number")) { if (setNumber.isEmpty()) setNumber = map.value("number").toString(); @@ -298,7 +292,7 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) rarity = map.value("rarity").toString(); } - extractColors(map.value("colors").toStringList(), colors); + colors << map.value("colors").toStringList(); } colors.removeDuplicates(); @@ -313,8 +307,8 @@ int OracleImporter::importTextSpoiler(CardSetPtr set, const QVariant &data) // add the card CardInfoPtr card = - addCard(set->getShortName(), cardName, false, muid, setNumber, cardCost, cmc, cardType, cardPT, cardLoyalty, - cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity); + addCard(set->getShortName(), cardName, false, muid, cardUuId, setNumber, cardCost, cmc, cardType, cardPT, + cardLoyalty, cardText, colors, relatedCards, reverseRelatedCards, upsideDown, rarity); if (!set->contains(card)) { card->addToSet(set); diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 538a327e2..68a7b62be 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -61,6 +61,7 @@ private: QString cardName, bool isToken, int cardId, + QString &cardUuId, QString &setNumber, QString &cardCost, QString &cmc, @@ -93,7 +94,6 @@ public: } protected: - void extractColors(const QStringList &in, QStringList &out); void sortColors(QStringList &colors); }; diff --git a/oracle/src/oraclewizard.cpp b/oracle/src/oraclewizard.cpp index 9cf24cd88..ef2445447 100644 --- a/oracle/src/oraclewizard.cpp +++ b/oracle/src/oraclewizard.cpp @@ -26,11 +26,22 @@ #include "settingscache.h" #include "version_string.h" -#define ZIP_SIGNATURE "PK" -#define ALLSETS_URL_FALLBACK "https://mtgjson.com/json/AllSets.json" +#ifdef HAS_LZMA +#include "lzma/decompress.h" +#endif #ifdef HAS_ZLIB #include "zip/unzip.h" +#endif + +#define ZIP_SIGNATURE "PK" +// Xz stream header: 0xFD + "7zXZ" +#define XZ_SIGNATURE "\xFD\x37\x7A\x58\x5A" +#define ALLSETS_URL_FALLBACK "https://mtgjson.com/json/AllSets.json" + +#ifdef HAS_LZMA +#define ALLSETS_URL "https://mtgjson.com/json/AllSets.json.xz" +#elif defined(HAS_ZLIB) #define ALLSETS_URL "https://mtgjson.com/json/AllSets.json.zip" #else #define ALLSETS_URL "https://mtgjson.com/json/AllSets.json" @@ -249,11 +260,14 @@ void LoadSetsPage::actLoadSetsFile() QFileDialog dialog(this, tr("Load sets file")); dialog.setFileMode(QFileDialog::ExistingFile); + QString extensions = "*.json"; #ifdef HAS_ZLIB - dialog.setNameFilter(tr("Sets JSON file (*.json *.zip)")); -#else - dialog.setNameFilter(tr("Sets JSON file (*.json)")); + extensions += " *.zip"; #endif +#ifdef HAS_LZMA + extensions += " *.xz"; +#endif + dialog.setNameFilter(tr("Sets JSON file (%1)").arg(extensions)); if (!fileLineEdit->text().isEmpty() && QFile::exists(fileLineEdit->text())) { dialog.selectFile(fileLineEdit->text()); @@ -383,7 +397,32 @@ void LoadSetsPage::readSetsFromByteArray(QByteArray data) progressBar->show(); // unzip the file if needed - if (data.startsWith(ZIP_SIGNATURE)) { + if (data.startsWith(XZ_SIGNATURE)) { +#ifdef HAS_LZMA + // zipped file + auto *inBuffer = new QBuffer(&data); + auto *outBuffer = new QBuffer(this); + inBuffer->open(QBuffer::ReadOnly); + outBuffer->open(QBuffer::WriteOnly); + XzDecompressor xz; + if (!xz.decompress(inBuffer, outBuffer)) { + zipDownloadFailed(tr("Xz extraction failed.")); + return; + } + + future = QtConcurrent::run(wizard()->importer, &OracleImporter::readSetsFromByteArray, outBuffer->data()); + watcher.setFuture(future); + return; +#else + zipDownloadFailed(tr("Sorry, this version of Oracle does not support xz compressed files.")); + + wizard()->enableButtons(); + setEnabled(true); + progressLabel->hide(); + progressBar->hide(); + return; +#endif + } else if (data.startsWith(ZIP_SIGNATURE)) { #ifdef HAS_ZLIB // zipped file auto *inBuffer = new QBuffer(&data); diff --git a/servatrice/CMakeLists.txt b/servatrice/CMakeLists.txt index 248030025..f1bf60925 100644 --- a/servatrice/CMakeLists.txt +++ b/servatrice/CMakeLists.txt @@ -164,6 +164,8 @@ if(WIN32) set(plugin_dest_dir Plugins) set(qtconf_dest_dir .) + install(DIRECTORY "${CMAKE_BINARY_DIR}/${PROJECT_NAME}/${CMAKE_BUILD_TYPE}/" DESTINATION ./ FILES_MATCHING PATTERN "*.dll") + # qt5 plugins: platforms, sqldrivers/mysql install(DIRECTORY "${QT_PLUGINS_DIR}/" DESTINATION ${plugin_dest_dir} COMPONENT Runtime FILES_MATCHING REGEX "(platforms/.*|sqldrivers/qsqlmysql)\\.dll"