diff --git a/.cspell/domain-specific.txt b/.cspell/domain-specific.txt new file mode 100644 index 0000000..219745b --- /dev/null +++ b/.cspell/domain-specific.txt @@ -0,0 +1,22 @@ +Aegisub +BadMutex +copas +CPATH +CTRF +DLUAJIT +DepCtrl +dkjson +FileOps +headlessly +leafo +libaegisub +luafilesystem +luajit +luarocks +luasocket +mimetypes +moonbase +moonc +moonpath +pegasus +PreciseTimer diff --git a/.cspell/ffi.txt b/.cspell/ffi.txt new file mode 100644 index 0000000..2490851 --- /dev/null +++ b/.cspell/ffi.txt @@ -0,0 +1,35 @@ +CONNECTTIMEOUT +CURLINFO +CURLMOPT +CURLMSG +CURLOPT +ENOENT +ENOTDIR +errornum +FAILONERROR +fclose +FOLLOWLOCATION +fopen +getinfo +gettime +getpid +nfds +NOPROGRESS +nsec +numfds +oflag +O_CREAT +pollfd +setopt +setx +strerror +syscall +timespec +trywait +uintptr +usec +usleep +USERAGENT +wchar +WinINet +WRITEDATA diff --git a/.cspell/lua.txt b/.cspell/lua.txt new file mode 100644 index 0000000..1e7defa --- /dev/null +++ b/.cspell/lua.txt @@ -0,0 +1,34 @@ +__newindex +addserver +addthread +chdir +collectgarbage +currentdir +finalizers +cdef +getenv +getmetatable +getsockname +gmatch +gsub +ipairs +lshift +luafilesystem +luajit +luajson +luarocks +metatable +metatables +newproxy +pcall +pcalls +randomseed +rawget +rshift +setmetatable +settimeout +setvbuf +tonumber +tostring +varargs +xpcall diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..8c0f14b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,61 @@ +name: Tests + +on: + pull_request: + branches: + - 'master' + workflow_dispatch: + inputs: + ref: + description: 'Release tag or branch to scan.' + required: true + default: 'master' + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.ref }} + + - uses: leafo/gh-actions-lua@v13 + with: + luaVersion: "luajit-2.1.0-beta3" + # DepCtrl relies on Lua 5.2 features (table.unpack, __pairs/__len) + luaCompileFlags: "XCFLAGS=-DLUAJIT_ENABLE_LUA52COMPAT" + + - uses: leafo/gh-actions-luarocks@v6 + # moonscript loader + - run: luarocks install moonscript + # lfs (Aegisub provides an internal copy) + - run: luarocks install luafilesystem + # CLI argument parsing + - run: luarocks install argparse + # mock HTTP server dependencies + - run: luarocks install luasocket + - run: luarocks install copas + - run: luarocks install pegasus + # json schema validation + - run: luarocks install lua-schema + - run: luarocks install lpeg + + - name: Run tests + timeout-minutes: 5 + run: lua depctrl.lua test + + - name: Publish test report + uses: ctrf-io/github-test-reporter@v1 + if: ${{ !cancelled() }} + with: + report-path: ./ctrf/*.json + pull-request: ${{ github.event_name == 'pull_request' && true || false }} + overwrite-comment: ${{ github.event_name == 'pull_request' && true || false }} + annotate: false + use-suite-name: true + status-check: true + summary-report: true + failed-report: true + upload-artifact: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ffd4d42 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/ctrf/*.json +/dist/ +/DependencyControl-*.zip diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..d5d0018 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "files.insertFinalNewline": true +} diff --git a/DependencyControl.json b/DependencyControl.json index 90690ce..65d4c35 100644 --- a/DependencyControl.json +++ b/DependencyControl.json @@ -1,5 +1,5 @@ { - "dependencyControlFeedFormatVersion": "0.3.0", + "dependencyControlFeedFormatVersion": "0.4.0", "name": "DependencyControl", "description": "The official DependencyControl repository.", "baseUrl": "https://github.com/TypesettingTools/DependencyControl", @@ -7,53 +7,64 @@ "fileBaseUrl": "https://raw.githubusercontent.com/TypesettingTools/DependencyControl/", "maintainer": "line0", "knownFeeds": { - "line0scripts": "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/DependencyControl.json", "a-mo": "https://raw.githubusercontent.com/TypesettingTools/Aegisub-Motion/DepCtrl/DependencyControl.json", - "SubInspector": "https://raw.githubusercontent.com/TypesettingTools/SubInspector/master/DependencyControl.json", + "arch1t3cht-scripts": "https://raw.githubusercontent.com/TypesettingTools/arch1t3cht-Aegisub-Scripts/main/DependencyControl.json", "ASSFoundation": "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json", + "coffeeflux-scripts": "https://raw.githubusercontent.com/TypesettingTools/CoffeeFlux-Aegisub-Scripts/master/DependencyControl.json", "ffi-experiments": "https://raw.githubusercontent.com/torque/ffi-experiments/master/DependencyControl.json", + "ILL": "https://raw.githubusercontent.com/TypesettingTools/ILL-Aegisub-Scripts/main/DependencyControl.json", + "line0scripts": "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/DependencyControl.json", "lyger-scripts": "https://raw.githubusercontent.com/TypesettingTools/lyger-Aegisub-Scripts/master/DependencyControl.json", - "unanimated-scripts": "https://raw.githubusercontent.com/TypesettingTools/unanimated-Aegisub-Scripts/master/DependencyControl.json", - "coffeeflux-scripts": "https://raw.githubusercontent.com/TypesettingTools/CoffeeFlux-Aegisub-Scripts/master/DependencyControl.json", "myaamori-scripts": "https://raw.githubusercontent.com/TypesettingTools/Myaamori-Aegisub-Scripts/master/DependencyControl.json", "petzku-scripts": "https://raw.githubusercontent.com/petzku/Aegisub-Scripts/master/DependencyControl.json", - "zahuczky-scripts": "https://raw.githubusercontent.com/Zahuczky/Zahuczkys-Aegisub-Scripts/main/DependencyControl.json", "phoscity-scripts": "https://raw.githubusercontent.com/PhosCity/Aegisub-Scripts/main/DependencyControl.json", - "zeref-scripts": "https://raw.githubusercontent.com/TypesettingTools/zeref-Aegisub-Scripts/main/DependencyControl.json", - "arch1t3cht-scripts": "https://raw.githubusercontent.com/TypesettingTools/arch1t3cht-Aegisub-Scripts/main/DependencyControl.json", - "ILL": "https://raw.githubusercontent.com/TypesettingTools/ILL-Aegisub-Scripts/main/DependencyControl.json" + "SubInspector": "https://raw.githubusercontent.com/TypesettingTools/SubInspector/master/DependencyControl.json", + "unanimated-scripts": "https://raw.githubusercontent.com/TypesettingTools/unanimated-Aegisub-Scripts/master/DependencyControl.json", + "zahuczky-scripts": "https://raw.githubusercontent.com/Zahuczky/Zahuczkys-Aegisub-Scripts/main/DependencyControl.json", + "zeref-scripts": "https://raw.githubusercontent.com/TypesettingTools/zeref-Aegisub-Scripts/main/DependencyControl.json" }, "macros": { "l0.DependencyControl.Toolbox": { - "url": "@{baseUrl}#@{namespace}", - "author": "line0", "name": "DependencyControl Toolbox", "description": "Provides DependencyControl maintenance and configuration utilities.", - "fileBaseUrl": "@{fileBaseUrl}macros-v@{version}-@{channel}/macros/@{namespace}", + "author": "line0", + "url": "@{baseUrl}#@{namespace}", + "fileBaseUrl": { + "script": "@{fileBaseUrl}macros-v@{version}-@{channel}/macros/@{namespace}", + "test": "@{fileBaseUrl}macros-v@{version}-@{channel}/macros/@{namespace}/test" + }, + "localFileBasePath": { + "script": "@{localFileBasePath}macros/@{namespace}", + "test": "@{localFileBasePath}macros/@{namespace}/test" + }, "channels": { "alpha": { - "version": "0.1.3", - "released": "2016-01-27", + "version": "0.3.0", + "released": null, "default": true, "files": [ { "name": ".moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "3677B2817C3D1FFE86981C8ABCC092B3D2CCEE7B" + "sha1": "C3DB67970FD16BC5A03CAA809E4901A0470F66E0" + }, + { + "name": ".moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "272D45C73246CC869106C705E217C8B32AE9D3FF", + "type": "test" } ], "requiredModules": [ { "moduleName": "l0.DependencyControl", - "version": "0.6.1" + "version": "0.7.0" } ] } }, "changelog": { - "0.1.0": [ - "initial release" - ], + "0.1.0": ["initial release"], "0.1.1": [ "The Install/Uninstall/Update dialogs now sort scripts by name.", "DependencyControl and its requirements no longer appear in the uninstall menu." @@ -63,97 +74,524 @@ ], "0.1.3": [ "Fixed an issue where trying to uninstall an unmanaged script resulted in an error unrelated to the intended error message." + ], + "0.2.0": [ + "Centralized automatic update scheduling for all installed scripts, including modules not loaded by any automation script, eliminating redundant per-environment checks at startup.", + "Unit test menus for installed modules with a DependencyControl test suite are now registered automatically at startup." + ], + "0.3.0": [ + "Added a Manage Feeds macro to review and manage the feeds DependencyControl knows about and their trust status: trust, block (with a reason), unblock, or remove feeds; discover feeds advertised by other feeds; and manage your extra feeds and block list in dedicated panels. It warns before an action would change an installed package's update source, opens feed details in the DepCtrl Browser, and won't let you block DependencyControl's own feed.", + "Manage Feeds now shows a Fetched column with how long ago each feed was last retrieved into the cache, read straight from the cache so it's visible even before Discover — letting you see at a glance which feeds have recent data.", + "The Toolbox now ships a DependencyControl unit test suite, adding it to the Run Tests menu." ] } } }, "modules": { "l0.DependencyControl": { - "url": "@{baseUrl}#@{namespace}", - "author": "line0", "name": "DependencyControl", "description": "Dependency manager and automatic script updater for Aegisub macros and modules.", - "fileBaseUrl": "@{fileBaseUrl}v@{version}-@{channel}/modules/@{scriptName}", + "author": "line0", + "url": "@{baseUrl}#@{namespace}", + "fileBaseUrl": { + "script": "@{fileBaseUrl}v@{version}-@{channel}/modules/@{namespacePath}", + "test": "@{fileBaseUrl}v@{version}-@{channel}/modules/@{namespacePath}/test" + }, + "localFileBasePath": { + "script": "@{localFileBasePath}modules/@{namespacePath}", + "test": "@{localFileBasePath}modules/@{namespacePath}/test" + }, "channels": { "alpha": { - "version": "0.6.3", - "released": "2016-02-06", + "version": "0.7.0", + "released": null, "default": true, "files": [ { "name": ".moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "76C22149258CB1189265A367C1B28046F54F8FB3" + "sha1": "A63135DFF9EED1AC3582279EA9E04EC5EC0DD399" + }, + { + "name": "/Common.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "695084AB63F2DFB4051E512147981CBD16D2DABF" + }, + { + "name": "/Constants.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "68762FCCFB4866C951F9506F1BBF2BBA37506FA0" }, { "name": "/ConfigHandler.moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "97BCD3207FE8158261FA7851057464535FCEFBC6" + "sha1": "572BDC2400E9C8A0E39B597EBF0AB6791AEF15B5" + }, + { + "name": "/ConfigView.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "583D90844CA4DE391C3D575AB62AD27B893C1421" + }, + { + "name": "/Crypto.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "3AE799E87F69321077CF7C05FC42FFCEADEF4A05" + }, + { + "name": "/Downloader.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "D2A85C738FEE492235928C44240FF81FEFA3BF39" + }, + { + "name": "/Enum.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "567FC08D8755935C9B595D0A6AF55BD1AEA405B1" + }, + { + "name": "/EventEmitter.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "ECD301C2EA0A74111981A6A0E2EAC6ADB2B32AB9" + }, + { + "name": "/FileCache.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "DE67AB15432485EE81D6CCB0FEC4AA039E5C7788" + }, + { + "name": "/FeedInventory.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "50926D379CD32DD00C334781A3F8F53DEAFD169E" + }, + { + "name": "/FeedManager.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "2F2D3DE5329929F7E996634B623BF85F622733B5" + }, + { + "name": "/FeedTrust.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "9FC31E9B83AFA1D6987A55243557A11528BD9696" + }, + { + "name": "/FileLock.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "F6CB3F535AC17E783687BA629FD6D672555AB41B" }, { "name": "/FileOps.moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "D999D34DB93BA76EF0E991CEB1CD63F5CC5F8E68" + "sha1": "E12FB6539B6CC7E12751B9C5AFD4E61ABDA60B2C" + }, + { + "name": "/GitRepository.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "2D23AAA1BD2A16555F3374253A36475114E2F650" + }, + { + "name": "/Host.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "DB00CB6086812586A3E1A096EE274870CF76BDB4" + }, + { + "name": "/JsonSchema.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "9E0E9AE5249B1B7112D4A5B6EE42CD80C42F6D19" + }, + { + "name": "/Lock.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "01C190CABEB961B7C4D79FF995B4815CFD0C8151" }, { "name": "/Logger.moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "1E479FE95F0DFBEE8B098302AB589F32D0C40A00" + "sha1": "DC505B34A15126541223065417AE3C9785C96104" + }, + { + "name": "/ModuleLoader.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "6576CB49736529DBD907028D1D2FAA1C0F3820A8" + }, + { + "name": "/ModuleProvider.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "1D0B293D563BE18FA2685966FC56E51683065FB8" + }, + { + "name": "/NamedSemaphore.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "CAD7BF0E840581EF16A62859A0A957CFADCFE57D" + }, + { + "name": "/Record.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "201CB6B4000043935398BC467B01682DF1566292" + }, + { + "name": "/ScriptTargetFilter.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "77CB9F63E63ACF0899E91B6B97698BE79D04B7F2" + }, + { + "name": "/ScriptUpdateRecord.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "8F236016F7ECC216C93ED5D99E7A27DA457D621B" + }, + { + "name": "/SemanticVersioning.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "5A5D88058A616584A2A9EC9EFFB5F8FAFD5575BF" + }, + { + "name": "/Stub.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "D28F02C65B29C6C29A6329A433E72E416E323356" + }, + { + "name": "/BadMutex.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "6B1C4E43BB588DDD2DE0FF730F3F288C641C535E", + "delete": true + }, + { + "name": "/Timer.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "BCE7AEB26FBCA73276DB61A0C86BFDD1F7569B95" }, { "name": "/UnitTestSuite.moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "ADAB6EFB05E08A7828DCA01BC1FC43D6482979A1" + "sha1": "C2C355B5622870FCA7E903375FE598682AC9D3C7" }, { "name": "/UpdateFeed.moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "1EE16D9D551FF82C2D7E448F2CD980E528874108" + "sha1": "A97EDD41EBA28C26265C00F5BFCCDF34D176FAE3" + }, + { + "name": "/UpdateTask.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "10558BBB9291B7B5B75061D6CE5FCC0705D62450" }, { "name": "/Updater.moon", "url": "@{fileBaseUrl}@{fileName}", - "sha1": "A4AE061724E68B2EFBB7495A477263E1746E228A" - } - ], - "requiredModules": [ + "sha1": "CC7E02780FF928FCFEBD283D76769D47A42A388E" + }, + { + "name": "/ZipArchiver.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "03F2629903107C97B514D3B85E8DA07268D6241B" + }, + { + "name": "/helpers/ffi-posix.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "972CE60ABA29D4BE64031C8E0FD087CA8EC59966" + }, + { + "name": "/helpers/ffi-windows.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "BF0B05684EF8B96B6657C60C4AE6B057A8CBDC55" + }, + { + "name": "/helpers/open-url.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "7F8313A28BF027859DA51871CADB384261D63F03" + }, + { + "name": "/helpers/resolve-host.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "5FCBB1F4EE1F919D8D23FFBFB70B4AFDF653E0AA" + }, + { + "name": "/shims/BadMutex.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "269B7FE920886A1A294A71305CFD4974F6EF3F48" + }, + { + "name": "/shims/DownloadManager.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "F5E25C2678ABFAA7055DB6F99E58E44F7C454E0A" + }, + { + "name": "/shims/PreciseTimer.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "F976BF883F90623D4D71191F1A5E72703A8551BE" + }, + { + "name": ".moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "EA95E00DC7D3E310C7D53B5E92C68936D82C9805", + "type": "test" + }, + { + "name": "/BadMutex.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "BDEA61A6DD65512495FD919754E8E14D351A48EE", + "type": "test" + }, + { + "name": "/Common.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "DDE1F2618560C2661B897B7A5210114626077F92", + "type": "test" + }, + { + "name": "/ConfigHandler.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "AA1FE5A7096924247B0C1DC746DD6A99A25BC5F3", + "type": "test" + }, + { + "name": "/ConfigView.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "87A74967602FF708D60D1D97F9A1CC24228150B6", + "type": "test" + }, + { + "name": "/Crypto.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "6E17511ED227B9423E331E7121A037828EB9EF18", + "type": "test" + }, + { + "name": "/Downloader.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "192E5AE5AA4D8E5583392F297A58545CBDCBFCE1", + "type": "test" + }, + { + "name": "/Enum.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "1BF9FF97C5668EB5046EE80B24F52F9C53F54A02", + "type": "test" + }, { - "moduleName": "requireffi.requireffi", - "version": "0.1.1", - "feed": "@{feed:ffi-experiments}" + "name": "/FileCache.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "675F23FE2B6A6D26280EB93FB6E9A9DA4B349A6A", + "type": "test" + }, + { + "name": "/FeedInventory.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "16AD4348385D81BFBF4591909A7DB9A913970130", + "type": "test" }, { - "moduleName": "DM.DownloadManager", - "version": "0.3.1", - "feed": "@{feed:ffi-experiments}" + "name": "/FeedManager.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "25F0A52E1A4B5E0E035772C636E0C67660A886C2", + "type": "test" }, { - "moduleName": "BM.BadMutex", - "version": "0.1.3", - "feed": "@{feed:ffi-experiments}" + "name": "/FeedTrust.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "3CC685E4472A2F667C7DDFAF4251F727F2962566", + "type": "test" }, { - "moduleName": "PT.PreciseTimer", - "version": "0.1.5", - "feed": "@{feed:ffi-experiments}" + "name": "/ffi-posix.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "535A053FE09841BDB4C757A77B7FB3EA360EA745", + "type": "test" + }, + { + "name": "/FileLock.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "FDC65E7FD7EB3190C2F7B888A6D48DD6CE3C1407", + "type": "test" + }, + { + "name": "/FileOps.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "1C054620FD0F2B139DC1772BE9D3E89EF19FD806", + "type": "test" + }, + { + "name": "/GitRepository.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "F7E43D74EF6574408DFD1B851EF40F18BDB64FFA", + "type": "test" + }, + { + "name": "/Host.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "8EAB0F381760D886D88C93787FD415CD429899A3", + "type": "test" + }, + { + "name": "/JsonSchema.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "7785C25F8060AD519EFA2BF9D9E41BF9D984A43A", + "type": "test" + }, + { + "name": "/Lock.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "B333BB9C78AFF5E3EF8CF5C5A94FFCA57185E937", + "type": "test" + }, + { + "name": "/Logger.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "5CBD204FCAEEDF428ECA40DE70883441B9A5E69C", + "type": "test" + }, + { + "name": "/ModuleLoader.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "EAB97CE45571F6A490AB5CAFBAE25C6C4B6F3C66", + "type": "test" + }, + { + "name": "/ModuleProvider.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "19B4A2217D237F9803DE6C6E1B6A486BE45EED73", + "type": "test" + }, + { + "name": "/NamedSemaphore.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "95C13F8E5A8005565BB7D2186BA52BB2C98A2935", + "type": "test" + }, + { + "name": "/open-url.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "A6ADC29FF30AB670AB2D0D8ED2A18740B591CD66", + "type": "test" + }, + { + "name": "/Record.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "2FDDE5146C44F276E0267835068A1430B82B464C", + "type": "test" + }, + { + "name": "/ScriptTargetFilter.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "278632008FE11ADB967B4761A2BFD1AB344C12B8", + "type": "test" + }, + { + "name": "/ScriptUpdateRecord.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "7576C5DC58A066F61AB3D173ADEC39C927C448CE", + "type": "test" + }, + { + "name": "/SemanticVersioning.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "93A0E08D48764E73366E7CB096C02836B609E017", + "type": "test" + }, + { + "name": "/Timer.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "9666CC4536FE09F6382EEDC5C2580280F7912111", + "type": "test" + }, + { + "name": "/UnitTestSuite.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "74C9D4D6CA22CD8570B6BA44720940DFB4CEF5CF", + "type": "test" + }, + { + "name": "/UpdateFeed.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "175CA233E43EFDFE148CA5DAFBB83A966366FAD8", + "type": "test" + }, + { + "name": "/UpdateTask.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "696ACA941219681F0F3391B438ACCF966D8544D0", + "type": "test" + }, + { + "name": "/Updater.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "0E5D0A3F4C50715F5C350AFC01EB2AC2A2671AA9", + "type": "test" + }, + { + "name": "/ZipArchiver.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "02A5F4F29EBE807A2F1FCB7BF8264CCB44F70FE5", + "type": "test" + }, + { + "name": "/integration/Downloader.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "70116D5787C65747AC6CFBD5D06BC484ED3B2A2B", + "type": "test" + }, + { + "name": "/integration/ZipArchiver.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "C719A5E37F1EDEFE995C49B7EFBFA4FFE708C8E9", + "type": "test" + }, + { + "name": "/helpers/mock-http-server.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "D6D87B5EA2F188EA5FF6D9479412FF5F9CE99EAE", + "type": "test" + }, + { + "name": "/helpers/MockHttpServerController.moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "C699B32FF02A9600C0C42DE90402EAB99A8A474A", + "type": "test" + } + ], + "provides": [ + { + "name": "BM.BadMutex", + "version": "^0.1.3" + }, + { + "name": "DM.DownloadManager", + "version": "^0.3.1" + }, + { + "name": "PT.PreciseTimer", + "version": "^0.1.6" } ] } }, "changelog": { - "0.6.3": [ - "Fixed a v0.6.2 regression that caused DependencyControl to fail loading the first time after a scheduled self-update." + "0.5.0": [ + "DependencyControl does now auto-update itself and its dependencies.", + "Provided Sub-Modules (Logger, ConfigHandler, ...) can now easily be accessed as class properties of the main DependencyControl module.", + "A bug was fixed that caused macros always being registered with the overall script description, ignoring specific descriptions for the macro menu entries.", + "The \\getConfigHandler() method no longer ignores the defaults parameter.", + "Fixed a FileOps bug that would cause path validation to fail on paths relative to the working directory.", + "ConfigHandler: writes to the configuration table are no longer accidentally routed to the defaults table when a value is updated that only exists in the Defaults.", + "ConfigHandler: Looping over the configuration table is now completely transparent wrt which fields are user configuration or defaults.", + "ConfigHandler: fixed a bug that prevented a global lock on the config file from being released on certain error conditions.", + "The update feed format has been updated to v0.2.0 and introduces a new template variable to reference knownFeeds specifed at the top level." ], - "0.6.2": [ - "An issue was fixed that would cause DepCtrl initializer code in modules previously loaded with regular Lua loading mechanisms to be skipped when requested in a _DependencyControl_- context. This kept the [requireffi](https://github.com/torque/ffi-experiments/tree/master/requireffi) _DependencyControl_ record from being established, preventing any updates from taking place.", - "UnitTestSuite: Fixed several broken assertions and related error messages, among them the `assertMatches` and `assertErrorMatches` assertions always returning `true`. Please make sure to rerun your tests after upgrading to confirm your tested actually return values whatever they were supposed to match.", - "Updater: Identical or duplicate feeds from different sources (user configuration, feeds and API use) are no longer being checked for updates multiple times.", - "Updater / FileOps: Fixed several broken error messages and return values." + "0.5.1": [ + "Macros registered using DependencyControl now get passed the previously missing 'active_line' parameter.", + "Fixed a bug that would cause an unrelated error to be thrown in place of the real error message when an updated module failed to load." ], - "0.6.1": [ - "The Updater component now supports the DownloadManager v0.4.0 API changes.", - "Updater: A regression introduced in v0.6.0 was fixed that caused update or installation processes to fail when the feed contained deletion records.", - "FileOps.mkdir() no longer falsely retuns an error state when a path to an existing file is passed with the `isFile` flag set." + "0.5.2": [ + "Updates and installations no longer fail when no suitable version of a module marked as an optional dependency can be found.", + "ConfigHandlers now recover gracefully when a corrupted config is encountered.", + "Fixed a bug that may have caused updates of unmanaged modules to throw an error after completion.", + "DependencyControl initialization functions in modules with optional DepCtrl support are now expected to use the predefined name __depCtrlInit. This lifts the unreasonable requirement of having to specify the name of the function in the dependency tables of the loading scripts. By extension, this also fixes errors when trying to update the binary modules required by DependencyControl (such as DownloadManager).", + "The Updater now checks for an active internet connection before going ahead with downloading feeds and packages.", + "FileOps: added a copy function for files." + ], + "0.5.3": [ + "ConfigHandler: A host of longstanding issues related to config file corruption and concurrent access to config files from multiple DepCtrl-hosting automation scripts have been fixed.", + "Error Reports of required modules loaded by DependencyControl now actually provide semi-useful stack traces.", + "A bug was fixed that could cause DepCtrl to rerun the __depCtrlInit method on modules even though a prior DependencyControl record had already been initialized.", + "The DependencyControl self-update now runs through properly without throwing an error at the end of the process." ], "0.6.0": [ "The UnitTestSuite framework for automatically testing automation scripts and modules has been added.", @@ -165,34 +603,104 @@ "FileOps.move() no longer fails to move files across file systems on *nix operating systems and properly cleans up after itself if files could not be overwritten and were renamed instead.", "FileOps: path validation is no longer broken on non-windows systems" ], - "0.5.3": [ - "ConfigHandler: A host of longstanding issues related to config file corruption and concurrent access to config files from multiple DepCtrl-hosting automation scripts have been fixed.", - "Error Reports of required modules loaded by DependencyControl now actually provide semi-useful stack traces.", - "A bug was fixed that could cause DepCtrl to rerun the __depCtrlInit method on modules even though a prior DependencyControl record had already been initialized.", - "The DependencyControl self-update now runs through properly without throwing an error at the end of the process." + "0.6.1": [ + "The Updater component now supports the DownloadManager v0.4.0 API changes.", + "Updater: A regression introduced in v0.6.0 was fixed that caused update or installation processes to fail when the feed contained deletion records.", + "FileOps.mkdir() no longer falsely retuns an error state when a path to an existing file is passed with the `isFile` flag set." ], - "0.5.2": [ - "Updates and installations no longer fail when no suitable version of a module marked as an optional dependency can be found.", - "ConfigHandlers now recover gracefully when a corrupted config is encountered.", - "Fixed a bug that may have caused updates of unmanaged modules to throw an error after completion.", - "DependencyControl initialization functions in modules with optional DepCtrl support are now expected to use the predefined name __depCtrlInit. This lifts the unreasonable requirement of having to specify the name of the function in the dependency tables of the loading scripts. By extension, this also fixes errors when trying to update the binary modules required by DependencyControl (such as DownloadManager).", - "The Updater now checks for an active internet connection before going ahead with downloading feeds and packages.", - "FileOps: added a copy function for files." + "0.6.2": [ + "An issue was fixed that would cause DepCtrl initializer code in modules previously loaded with regular Lua loading mechanisms to be skipped when requested in a _DependencyControl_- context. This kept the [requireffi](https://github.com/torque/ffi-experiments/tree/master/requireffi) _DependencyControl_ record from being established, preventing any updates from taking place.", + "UnitTestSuite: Fixed several broken assertions and related error messages, among them the `assertMatches` and `assertErrorMatches` assertions always returning `true`. Please make sure to rerun your tests after upgrading to confirm your tested actually return values whatever they were supposed to match.", + "Updater: Identical or duplicate feeds from different sources (user configuration, feeds and API use) are no longer being checked for updates multiple times.", + "Updater / FileOps: Fixed several broken error messages and return values." ], - "0.5.1": [ - "Macros registered using DependencyControl now get passed the previously missing 'active_line' paramter.", - "Fixed a bug that would cause an unrelated error to be thrown in place of the real error message when an updated module failed to load." + "0.6.3": [ + "Fixed a v0.6.2 regression that caused DependencyControl to fail loading the first time after a scheduled self-update." ], - "0.5.0": [ - "DependencyControl does now auto-update itself and its dependencies.", - "Provided Sub-Modules (Logger, ConfigHandler, ...) can now easily be accessed as class properties of the main DependencyControl module.", - "A bug was fixed that caused macros always being registered with the overall script description, ignoring specific descriptions for the macro menu entries.", - "The \\getConfigHandler() method no longer ignores the defaults parameter.", - "Fixed a FileOps bug that would cause path validation to fail on paths relative to the working directory.", - "ConfigHandler: writes to the configuration table are no longer accidentally routed to the defaults table when a value is updated that only exists in the Defaults.", - "ConfigHandler: Looping over the configuration table is now completely transparent wrt which fields are user configuration or defaults.", - "ConfigHandler: fixed a bug that prevented a global lock on the config file from being released on certain error conditions.", - "The update feed format has been updated to v0.2.0 and introduces a new template variable to reference knownFeeds specifed at the top level." + "0.6.4": [ + "Logger: Fixed a crash when `logEx()` is called without format arguments — `msg:format(...)` is now skipped when no varargs are supplied.", + "Logger: `fileBaseName` now falls back to `\"UNKNOWN\"` when `script_namespace` is nil, preventing errors during Logger initialization in contexts where no namespace is available.", + "Logger/UpdateFeed: Fixed chained method calls on file handles (`handle:write():flush()` and `handle:write():close()`) that could silently swallow errors" + ], + "0.7.0": [ + "The previously monolithic `DependencyControl.moon` has been broken up into focused sub-modules as groundwork for a future SQLite-based script registry backend: `Record` (version record management), `ModuleLoader` (module loading and dependency resolution), `SemanticVersioning` (version number handling), and `Common` (shared enums and utilities).", + "Script types (automation macros vs. modules) and record types are now represented by proper enums (`ScriptType`, `RecordType`) instead of bare booleans, making the API more explicit and extensible.", + "UpdateFeed: Fixed two regressions caused by the refactoring, both of which caused the update process to fail.", + "Global initialization has been moved into a dedicated setup method, reducing implicit global state for loggers and configuration.", + "DepCtrl now refuses to load if the installed Moonscript is below the minimum required version with a helpful error message directing users to update their Aegisub build.", + "ModuleLoader: Fixed a regression where DepCtrl init hooks were called again on already-initialized modules, causing errors in modules that mutate their exported state on first call (e.g. BadMutex).", + "Common: Fixed a long-standing bug that guaranteed the `capitalize()` function to fail, that was never caught because it was unused until the refactoring.", + "Common: Added the `makeSet()`, `addDefaults()`, and `trim()` utility helpers (building a set from a list, filling in missing table defaults, and stripping surrounding whitespace from a string), shared with the Functional module.", + "Updater: Fixed a potential issue where a multi-assignment statement could corrupt record fields after an unsuccessful update.", + "Automatic update scheduling is now centralized in the DependencyControl Toolbox and runs in a single Aegisub environment at startup, covering all installed scripts including modules not loaded by any automation script. Previously, each automation script's environment scheduled redundant checks for every module it loaded.", + "Record: Added `getAllRegisteredRecords()` to expose the full live record registry to tooling.", + "Modules with a DependencyControl unit test suite now have their test menus registered automatically by the Toolbox. Automation scripts register their own test menus when they call `registerMacros` through DependencyControl.", + "Automation scripts can now ship a DependencyControl unit test suite, just like modules. `registerMacros` accepts an optional table of test exports (the script's internal values to test), which — together with the script's registered macros, now exposed to the suite by name — is passed to the suite's import function.", + "Lock: Locks are now per-resource — distinct namespace/resource pairs can be held at the same time instead of contending over a single global mutex — and accept an optional `Global` scope that enforces mutual exclusion across separate Aegisub instances (now used when reading and writing shared config files). A Global lock is backed by an OS advisory file lock that the system releases automatically if the holder crashes, so a crashed process can never leave a config file permanently locked.", + "Lock: While a lock is held it records its holder (name, process id, and lease expiry) in a side file for troubleshooting, and logs a warning when it is waiting on a holder whose lease has lapsed (a likely crash or stall). The recorded lease is honored by waiters; an `overrideExpiry` option lets a waiter apply its own expiry instead.", + "Lock: Added `renew(threshold)` to extend a held lock's lease during long operations (refreshing only when the remaining lease drops below the threshold, so it is cheap to call from a busy loop), and a `Lock.guard` helper that acquires a lock, runs a function, and always releases it (even if the function errors).", + "Updater: The in-progress-update flag is no longer stored in the config file; concurrent updates are now coordinated by a dedicated cross-process lock that the system releases automatically if an updater crashes. The lease is renewed throughout long downloads, and a new `Updater.isRunning()` reports whether (and which script) an update is currently running.", + "Timer: Exposed a shared monotonic `Timer.getTime()` clock, added stopwatch-style `start()`, `stop()`, and `reset()` methods, and PT.PreciseTimer is now provided through the same bundled-provider mechanism as BM.BadMutex and DM.DownloadManager (set DEPCTRL_FORCE_BUILTIN_TIMER=1 to force DepCtrl's implementation).", + "Updater: Scripts and modules installed in Aegisub's data automation directory (e.g. native modules installed by a system package manager) are no longer updated by DependencyControl — scheduled checks skip them with a debug log, while manual installs/updates and dependency resolution report a descriptive error. Portable / \"Local Config\" setups, where the user and data directories are the same, are unaffected.", + "Updater: A required module that no feed offers directly can now be satisfied by installing a module that declares it in `provides` — for example, a script requiring `json` is satisfied by installing `dkjson`. Providers are only used when no feed offers the module by name; a candidate must be installable on your platform and meet the required version, and a directly-named module is always preferred over a provider. Once a provider is installed for a requirement, it keeps being used and updated instead of switching to a different provider (as long as it can still satisfy the requirement). Feeds can now advertise `provides` per release, and `update-feed` mirrors a module's declared aliases into the feed automatically. A provider can pin a provided alias to an npm-style version range (e.g. `~1.2`), so it keeps satisfying requirements within that range without having to be re-published for every patch release of the provided module.", + "Updater: Package sources are now selected by trust to harden against supply-chain attacks. Feeds are consulted closest-first (the feed a script declares for a dependency, then feeds DependencyControl trusts — those in its own feed plus your `extraFeeds` — then any others), and the install is refused if a requirement can only be met from an untrusted feed. Your `extraFeeds` are discovery roots that are also searched for updates, while your `trustedFeeds` only grant trust to feeds reached another way and are never themselves searched. A new `blockedFeeds` config (and an official block list in DependencyControl's own feed) lets a compromised feed be rejected outright. The obsolete `tryAllFeeds` setting and its exhaustive update mode have been removed.", + "Updater: When an install would otherwise be refused because it can only be satisfied from an untrusted feed, DependencyControl can now ask you to confirm and either use it just this once, add it to your trusted feeds, or block it outright; and when several equally-ranked packages can satisfy a requirement it can ask you to pick one. How freely it may prompt is controlled by two new settings: `feedTrustPromptThreshold` for trusting an untrusted feed (defaults to prompting in all situations, since it decides whether an update succeeds) and `packageChoicePromptThreshold` for picking between equally-ranked packages (defaults to only actions you start yourself).", + "Updater: The package picker now also lets you decide how long your choice should stick — use the source only this once, remember it while it stays available, pin it so DependencyControl never switches silently, or hand the decision back to DependencyControl for good. The remembered choice is stored per package and follows the source across feed-URL changes; a pinned source that becomes unavailable fails a required dependency (so you can fix the pin) and skips an optional one. A new global `offerAllSources` setting offers every eligible source (including lower-ranked and untrusted ones) whenever there's more than one, so you can override the ranking.", + "Unit test framework: `assertContains` now honors its case-insensitive option (it previously always compared case-sensitively) and reports a proper message when the assertion fails.", + "Unit test framework: a failing test-class setup is now reported as a failure instead of aborting the entire test run with an internal error.", + "Unit test framework: a failing `assertNegative` now says \"negative\" rather than \"positive\" in its message.", + "Updater: the temporary download directory is now removed after a successful install instead of being left behind.", + "FileOps: `mkdir` no longer reports success when the directory could not actually be created.", + "Logger: the download progress bar now fills incrementally instead of jumping from empty to full at the halfway point.", + "Unit test framework: a failing `assertPositive`/`assertNegative` no longer errors while formatting its failure message.", + "Disabling updates is now fully honored: it also prevents installing or updating modules on demand, not just scheduled background update checks.", + "Updater: Feed and package downloads now refuse hosts that resolve to a private, loopback, or link-local address (an SSRF safeguard), controlled by the new `blockPrivateHosts` setting (on by default; turn it off to use feeds on your local network).", + "Added `FeedInventory` and `FeedManager` — building blocks that list the feeds DependencyControl knows about, each with its provenance and trust status, and apply feed-trust actions. They back the Toolbox's new Manage Feeds macro and are available to other tooling.", + "Feed discovery can now follow feeds' advertised `knownFeeds` transitively to surface feeds beyond the ones you've configured, with untrusted expansion bounded so a hostile feed can't drive unbounded fetches (new `fetchUntrustedFeeds` and `crawlLimits` settings).", + "Fetched feeds are now cached to disk as timestamped snapshots (under `?user/cache/l0.DependencyControl/feeds`), so a recently-fetched feed is served from the cache instead of being re-downloaded, and a failed fetch falls back to the last cached copy for offline resilience. The freshness window is configurable via the new `cacheMaxAge` setting and the location via the shared `cache` path, replacing the former `dumpFeeds` debug dump.", + "DependencyControl's settings file is now organized into topic sections (`updates`, `feeds`, `logging`, `paths`), and an existing config is migrated into the new layout automatically on first load, so no manual edits are needed. Several settings were renamed for consistency in the process (e.g. `updaterEnabled` became `updates.mode`, `updaterBlockPrivateHosts` became `updates.blockPrivateHosts`, and `feedCacheMaxAge` became `feeds.cacheMaxAge`), and the separate feed-cache directory gave way to a single `paths.cache` location that DependencyControl organizes by namespace internally.", + "UpdateFeed: a file marked for deletion in a feed is no longer reported as a missing-source error when deploying, and is now removed from the output directory when present.", + "Updater: The on/off switch grew into an update mode: `updates.mode` takes 'off', 'user-requested', 'dependency-resolution', or 'auto-update' to control which contexts may install and update — for example, disable background update checks while still allowing manual installs and dependency resolution. An existing `updaterEnabled` setting migrates automatically ('auto-update' when true, 'off' when false). The two prompt-threshold settings now take the same context values in place of their former numeric levels, plus 'off' to disable a prompt kind entirely." + ] + } + }, + "l0.dkjson": { + "name": "dkjson", + "description": "David Kolf's JSON module for Lua, vendored with and managed by DependencyControl.", + "author": "David Kolf", + "url": "http://dkolf.de/dkjson-lua/", + "fileBaseUrl": "@{fileBaseUrl}v@{version}-@{channel}/modules/@{namespacePath}", + "localFileBasePath": "@{localFileBasePath}modules/@{namespacePath}", + "channels": { + "release": { + "version": "2.10.0", + "released": null, + "default": true, + "files": [ + { + "name": ".moon", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "73EDA9FC07D0F6B4B3713F08262C337B520B69B9" + }, + { + "name": "/vendor/dkjson.lua", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "A0597E1AEEB14D42DABB3A0E8C05129EB024EDCA" + } + ], + "provides": [ + { + "name": "json" + }, + { + "name": "dkjson" + } + ] + } + }, + "changelog": { + "2.10.0": [ + "Vendored dkjson v2.10 with a DependencyControl version record and json/dkjson self-registration." ] } } diff --git a/README.md b/README.md index b7d2834..d92b70b 100644 --- a/README.md +++ b/README.md @@ -1,97 +1,151 @@ -DependencyControl - Enterprise Aegisub Script Management --------------------------------------------------------- +# DependencyControl - Enterprise Aegisub Script Management DependencyControl provides versioning, automatic script update, dependency management and script management services to Aegisub macros and modules. -__Features__: +**Features**: - * A lightweight package manager lets users conveniently install scripts right from inside Aegisub - * Loads modules used by an automation script, pulls missing requirements from the internet and informs the user about missing and outdated modules that could not be updated automatically - * Checks scripts and modules for updates and automatically installs them - * Offers convenient macro registration with user-customizable submenus - * Provides configuration, logging services, file operations and a unit test framework for your scripts - * Supports optional modules and private module copies for cases where an older or custom version of a module is required - * Resolves circular dependencies (limitations apply) +- A lightweight package manager lets users conveniently install scripts right from inside Aegisub +- Loads modules used by an automation script, pulls missing requirements from the internet and informs the user about missing and outdated modules that could not be updated automatically +- Checks scripts and modules for updates and automatically installs them +- Offers convenient macro registration with user-customizable submenus +- Provides configuration, logging services, file operations and a unit test framework for your scripts +- Supports optional modules and private module copies for cases where an older or custom version of a module is required +- Resolves circular dependencies (limitations apply) -__Requirements__: +**Requirements**: - * Aegisub > 3.2.0 (e.g. [Plorkyeran's](http://plorkyeran.com/aegisub/) r8792+ or [my](http://files.line0.eu/builds/Aegisub/) git builds) - * [LuaJSON](https://github.com/harningt/luajson) - * [DownloadManager](https://github.com/torque/ffi-experiments/releases) v0.3.0 - * [BadMutex](https://github.com/torque/ffi-experiments/releases) v0.1.2 - * [PreciseTimer](https://github.com/torque/ffi-experiments/releases) v0.1.4 +- Aegisub [v3.4.0+](https://github.com/TypesettingTools/Aegisub/releases) or releases of [arch1t3cht's Aegisub fork](https://github.com/arch1t3cht/Aegisub/releases) based on v3.4.0+. Older versions of Aegisub may work, but you're on your own if you run into any issues. ----------------------------------- +DependencyControl is self-contained: it bundles a JSON library ([dkjson](https://dkolf.de/dkjson-lua/)), though if you have another `json` module installed, it is used instead. +It also now ships with pure-FFI +implementations of functionality previously provided by +[ffi-experiments](https://github.com/torque/ffi-experiments) +modules (_DownloadManager_, _BadMutex_, _PreciseTimer_). -### Documentation ### +--- - 1. [DependencyControl for Users](#dependency-control-for-users) - 2. [Usage for Automation Scripts](#usage-for-automation-scripts) - 3. [Namespaces and Paths](#namespaces-and-paths) - 4. [The Anatomy of an Updater Feed](#the-anatomy-of-an-updater-feed) - 5. [Reference](#reference) - 1. [DependencyControl](#FIXME) - 2. [Updater](#FIXME) - 3. [Logger](#FIXME) - 4. [ConfigHandler](#FIXME) - 5. [FileOps](#FIXME) +## Table of Contents ----------------------------------- +1. [DependencyControl for Users](#dependency-control-for-users) +2. [Usage for Automation Scripts](#usage-for-automation-scripts) +3. [Namespaces and Paths](#namespaces-and-paths) +4. [The Anatomy of an Updater Feed](#the-anatomy-of-an-updater-feed) +5. [Reference](#reference) +6. [DependencyControl](#FIXME) +7. [Updater](#FIXME) +8. [Logger](#FIXME) +9. [ConfigHandler](#FIXME) +10. [FileOps](#FIXME) +11. [CLI](#cli) -### Dependency Control for Users ### +--- + +## Dependency Control for Users As an end-user you don't get to decide whether your scripts use DependencyControl or not, but you can control many aspects of its operation. The updater works out-of-the-box (for any script with an update feed) and is run automatically. -#### Install Instructions #### - 1. Download the latest DependencyControl release for your platform and unpack its contents to your Aegisub **user** automation directory. - Alternatively use one of the [provided Aegisub builds](http://files.line0.eu/builds/Aegisub/) with built-in DependencyControl. +### Installation + +1. Download the latest DependencyControl release unpack its contents to your Aegisub **user** automation directory: + - On Windows: `%AppData%\Aegisub\automation` + - On Linux: `~/.aegisub/automation` + - On OSX: `~/Library/Application Support/Aegisub/automation` + +Do **NOT** unpack the file into the automation directory within the Aegisub installation folder, as this will break the updater. - _It is essential DependencyControl and all scripts it's used reside in the **user** automation directory, **NOT** the the automation directory in the Aegisub application folder._ +2. Restart Aegisub or re-scan your autoload directory from within the Aegisub _Automation Manger_. - On Windows, this will be `%AppData%\Aegisub\automation` folder. +### Configuration -2. In Aegisub, rescan your automation folder (or restart Aegisub). +DependencyControl comes with sane default settings, so if you're happy with that, there's no need to read further. If you want to disable the updater, use custom menus or want to tweak another aspect of DependencyControl, read on. -#### Configuration #### -DependencyControl comes with sane default settings, so if you're happy with that, there's no need to read further. If you want to disable the updater, use custom menus or want to tweak another aspect of DepedencyControl, read on. +DependencyControl stores its configuration as a JSON file in the `config` folder of your Aegisub user directory: -DependencyControl stores its configuration as a JSON file in the _config_ subdirectory of your Aegisub folder (`l0.DependencyControl.json`). Currently you'll have to edit this file manually, in the future there will be a management macro. +- On Windows: `%AppData%\Aegisub\config\l0.DependencyControl.json` +- On Linux: `~/.aegisub/config/l0.DependencyControl.json` +- On OSX: `~/Library/Application Support/Aegisub/config/l0.DependencyControl.json` + +The **DependencyControl Toolbox** macro provides a GUI for common management tasks; advanced options still require manual JSON editing. There are 2 kinds of configuration: -##### 1. Global Configuration ##### +#### 1. Global Configuration + Changes made in the `config` section of the configuration file will affect all scripts and general DependencyControl behavior. -__Available Fields__: - -* *bool* __updaterEnabled [true]:__ Turns the updater on/off -* *int* __updateInterval [3 Days]:__ The time in seconds between two update checks of a script -* *int* __traceLevel [3]:__ Sets the Trace level of DependencyControl update messages. Setting this higher than your _Trace level_ setting in Aegisub will prevent any of the messages from littering your log window. -* *bool* __dumpFeeds [true]:__ Debug option that will make DependencyControl dump updater feeds (original and expanded) to your Aegsiub folder. -* *arr* __extraFeeds:__ lets you provide additional update feeds that will be used when checking any script for updates -* *bool* __tryAllFeeds [false]:__ When set to true, feeds available to update a macro or module will be checked until an update is found. When set to false, a regular update process will stop once a feed confirms the script to be up-to-date. -* *str* __configDir ["?user/config"]:__ Sets the configuration directory that will be "offered" to automation scripts (they may or may not actually use it) -* *str* __writeLogs [true]:__ When enabled, DependencyControl log messages will be written to a file in the Aegisub log folder. This is a valuable resource for debugging, especially since the Aegisub log window is not available during script initalization. -* *int* __logMaxFiles [200]:__ DepedencyControl will purge old updater log files when any of the limits for log file count, log age and cumulative file size is exceeded. -* *int* __logMaxAge [1 Week]:__ Logs with a last modified date that exceeds this limit will be deleted. Takes a duration in seconds. -* *int* __logMaxSize [10 MB]:__ Cumulative file size limit for all log files in bytes. - -##### 1. Per-script Configuration ##### +**Available Fields**: + +- _str_ **updates.mode ["auto-update"]:** Which update contexts may install and update at all. Each value includes the ones before it: + - `"off"` — no installs or updates at all + - `"user-requested"` — only actions you start yourself (e.g. via the Toolbox) + - `"dependency-resolution"` — also installing/updating modules a script depends on + - `"auto-update"` (default) — also background scheduled update checks +- _int_ **updateInterval [3 Days]:** The time in seconds between two update checks of a script +- _int_ **traceLevel [3]:** Sets the Trace level of DependencyControl update messages. Setting this higher than your _Trace level_ setting in Aegisub will prevent any of the messages from littering your log window. +- _bool_ **dumpFeeds [true]:** Debug option that will make DependencyControl dump updater feeds (original and expanded) to your Aegisub folder. +- _arr_ **extraFeeds:** lets you provide additional update feeds that will be used when checking any script for updates. Feeds you list here are treated as trusted. +- _arr_ **trustedFeeds:** additional feed URLs you trust as package sources, on top of the feeds DependencyControl trusts by default (those advertised in its own feed). Unlike `extraFeeds`, these aren't crawled for updates on their own — they only mark a feed as trusted so packages can be installed from it without a warning. +- _arr_ **blockedFeeds:** feed URLs that must never be used as a package source. A blocked feed is rejected regardless of any other setting (including `userFeed`). Applied on top of DependencyControl's own block list. Each entry is matched case-insensitively as a URL prefix, so a host root like `https://example.com/` blocks every feed under it. +- _str_ **feedTrustPromptThreshold ["auto-update"]:** When DependencyControl may ask you to approve installing from a feed not (yet) on the trusted list. Takes the same context values as `updates.mode` (`"off"` never asks; defaults to `"auto-update"`, i.e. asking is allowed in every context). When a situation isn't allowed to prompt, the install fails or is skipped rather than using the untrusted feed. +- _str_ **packageChoicePromptThreshold ["user-requested"]:** When DependencyControl may ask you to pick between equally-ranked packages that can satisfy a requirement. Same context values as above (default `"user-requested"`: only for actions you start yourself). When a situation isn't allowed to prompt, a stable but arbitrary tie-breaker is used. +- _bool_ **packageChoiceOfferAllSources [false]:** By default the package picker only appears when two or more sources are genuinely tied (same trust band and version). Set this to `true` to be offered _every_ eligible source whenever there's more than one (including lower-ranked and untrusted ones). +- _str_ **configDir ["?user/config"]:** Sets the configuration directory that will be "offered" to automation scripts (they may or may not actually use it) +- _str_ **writeLogs [true]:** When enabled, DependencyControl log messages will be written to a file in the Aegisub log folder. This is a valuable resource for debugging, especially since the Aegisub log window is not available during script initialization. +- _int_ **logMaxFiles [200]:** DependencyControl will purge old updater log files when any of the limits for log file count, log age and cumulative file size is exceeded. +- _int_ **logMaxAge [1 Week]:** Logs with a last modified date that exceeds this limit will be deleted. Takes a duration in seconds. +- _int_ **logMaxSize [10 MB]:** Cumulative file size limit for all log files in bytes. + +#### 2. Per-script Configuration + Changes made in the `macros` and `modules` sections of the configuration file affect only the script or module in question. -__Available Fields__: +**Available Fields**: + +- _str_ **customMenu:** If you want to sort your automation macros into submenus, set this to the submenu name (use `/` to denote submenu levels). +- _str_ **userFeed:** When set the updater will use this feed exclusively to update the script in question (instead of other feeds) +- _int_ **lastUpdateCheck [auto]:** This field is used to store the (epoch) time of the last update check. +- _obj_ **currentSource [auto]:** Remembers which source last satisfied this package and how sticky that choice is (see [Remembering your source choice](#remembering-your-source-choice)). Managed automatically; don't edit it. +- _int_ **logLevel [3]:** sets the default trace level for log messages from this script (only applies to messages sent through a Logger instance provided by DependencyControl to the script) +- _bool_ **logToFile [false]:** set the user preference wrt/ whether log messages of this script should be written to disk or not (same restrictions as above apply, may be overridden by the script) +- `author`, `configFile`, `feed`, `moduleName`, `name`, `namespace`, `url`, `requiredModules`, `version`, `unmanaged`, `provides`: These fields hold aspects of the script's version record. Don't change them (they will be reset anyway) + +### How DependencyControl Selects Package Sources + +When a script or module needs to be installed or updated, more than one feed may be able to supply it. DependencyControl chooses the source by **trust first, version second**, so an unexpected or compromised feed can't win just by advertising a higher version number. Candidates are ranked best-first: + +1. **The package's own / declared feed**: the feed an installed package advertises, or the feed the depending script declares for the dependency — as long as it still offers the package and remains trusted. +2. **Other trusted feeds**: offering the package directly by name. +3. **Trusted feeds offering a _provider_**: a different module that declares it `provides` the required name (see [Providing module aliases](#providing-module-aliases)). +4. **Untrusted feeds** (by name, then via a provider): interactive installs ask for confirmation before using them. Silent installs/updates are refused until the feed is added to the trusted list. + +Within a tier the highest satisfying version wins. Feeds are fetched lazily in this order, so closer, more-trusted sources are tried before DependencyControl reaches further out. -* *str* __customMenu:__ If you want to sort your automation macros into submenus, set this to the submenu name (use `/` to denote submenu levels). -* *str* __userFeed:__ When set the updater will use this feed exclusively to update the script in question (instead of other feeds) -* *int* __lastUpdateCheck [auto]:__ This field is used to store the (epoch) time of the last update check. -* *int* __logLevel [3]:__ sets the default trace level for log messages from this script (only applies to messages sent through a Logger instance provided by DepedencyControl to the script) -* *bool* __logToFile [false]:__ set the user preference wrt/ whether log messages of this script should be written to disk or not (same restrictions as above apply, may be overridden by the script) -* author, configFile, feed, moduleName, name, namespace, url, requiredModules, version, unmanaged: These fields hold aspects of the script's version record. Don't change them (they will be reset anyway) +#### Customizing Feed Sources & Trust ------------------------------------------ -### Usage for Automation Scripts ### +The feeds DependencyControl advertises in its own feed are trusted out of the box, as is anything you add yourself. Tune this in your [global configuration](#1-global-configuration): -#### For Macros: #### +- **`trustedFeeds`**: additional feed URLs you trust as package sources. +- **`extraFeeds`**: extra feeds to check for updates (these count as trusted, too). +- **`blockedFeeds`**: feeds that must never be used, overriding everything else (applied in addition to DependencyControl's own block list). Entries match case-insensitively by URL prefix, so a host root blocks every feed under it. +- **`userFeed`** (per script): pin a single feed to be used exclusively for that script. + +(A future DependencyControl Toolbox UI will let you confirm and trust feeds interactively instead of editing the config by hand.) + +#### Remembering your source choice + +When a package can be installed from more than one source and DependencyControl asks you to pick one, the picker also lets you decide how long that choice should stick. The buttons map to: + +- **Just This Once**: use the picked source for this install only; you'll be asked again next time there's a choice. +- **Remember**: keep using the picked source as long as it stays available. If it later disappears (e.g. the feed stops offering the package), DependencyControl asks you to choose again — or, during a non-interactive update, falls back to asking next time and proceeds with its own ranked pick for now. +- **Pin/Lock**: pin this source. DependencyControl will keep using it without asking and will refuse to silently switch — if a pinned source becomes unavailable, a required dependency fails (so you can fix the pin) while an optional one is skipped. +- **Auto-Pick**: stop asking for this package and always take the top-ranked source automatically. +- **Cancel**: cancel the operation without installing. + +A remembered choice follows the source even if its feed URL changes, so moving a feed won't break it. When the picked source is reached through a [provider](#providing-module-aliases), it's the provider that's remembered, so version bumps update it in place instead of switching to a different one. + +## Usage for Automation Scripts + +### For Macros Load DependencyControl at the start of your macro and create a version record. Script and version information is automatically pulled from the `script_*` variables (the additional `script_namespace` variable is **required**). @@ -106,137 +160,183 @@ script_namespace = "l0.MoveAlongPath" local DependencyControl = require("l0.DependencyControl") local version = DependencyControl{ - feed = "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/DependencyControl.json", - { - "aegisub.util", - {"a-mo.LineCollection", version="1.0.1", url="https://github.com/torque/Aegisub-Motion"}, - {"a-mo.Line", version="1.0.0", url="https://github.com/TypesettingTools/Aegisub-Motion"}, - {"a-mo.Log", url="https://github.com/torque/Aegisub-Motion"}, - {"l0.ASSFoundation", version="0.1.1", url="https://github.com/TypesettingTools/ASSFoundation", - feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"}, - {"l0.ASSFoundation.Common", version="0.1.1", url="https://github.com/TypesettingTools/ASSFoundation", - feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"}, - "YUtils" - } + feed = "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/DependencyControl.json", + { + "aegisub.util", + {"a-mo.LineCollection", version="1.0.1", url="https://github.com/torque/Aegisub-Motion"}, + {"a-mo.Line", version="1.0.0", url="https://github.com/TypesettingTools/Aegisub-Motion"}, + {"a-mo.Log", url="https://github.com/torque/Aegisub-Motion"}, + {"l0.ASSFoundation", version="0.1.1", url="https://github.com/TypesettingTools/ASSFoundation", + feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"}, + {"l0.ASSFoundation.Common", version="0.1.1", url="https://github.com/TypesettingTools/ASSFoundation", + feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"}, + "YUtils" + } } local util, LineCollection, Line, Log, ASS, Common, YUtils = version:requireModules() ``` -Specifying a feed in your own version record provides DepedencyControl with a source to download updates to your script from. +Specifying a feed in your own version record provides DependencyControl with a source to download updates to your script from. Specifying feeds for required modules managed by DependencyControl allows the Updater to discover those modules and fetch them when they're missing from the user's computer. However, you can omit the feed URLs for required modules when your own feed already has references to them. +To **register your macros** use the following code snippets instead of the usual _aegisub.register_macro()_ calls: -To __register your macros__ use the following code snippets instead of the usual *aegisub.register_macro()* calls: +For a **single macro** that should be registered using the _script_name_ as automation menu entry, use: -For a __single macro__ that should be registered using the *script_name* as automation menu entry, use: ```Lua version:registerMacro(myProcessingFunction) ``` -For a script that registers __several macros__ using its own submenu use: +For a script that registers **several macros** using its own submenu use: + ```Lua version:registerMacros{ - {script_name, "Opens the Move Along Path GUI", showDialog, validClip}, - {"Undo", "Reverts lines to their original state", undo, hasUndoData} + {script_name, "Opens the Move Along Path GUI", showDialog, validClip}, + {"Undo", "Reverts lines to their original state", undo, hasUndoData} } ``` -Using this method for macro registration is a requirement for the __custom submenus__ feature to work with your script and lets DependencyControl hook your macro processing function to run an update check when your macro is run. +Using this method for macro registration is a requirement for the **custom submenus** feature to work with your script and lets DependencyControl hook your macro processing function to run an update check when your macro is run. -#### For Modules: #### +### For Modules -Creating a record for a module is very similar to how it does for macros, with the key difference being that name and version information is passed to DependencyControl correctly and a *moduleName* is required. +Creating a record for a module is very similar to how it does for macros, with the key difference being that name and version information is passed to DependencyControl correctly and a _moduleName_ is required. ```lua - local DependencyControl = require("l0.DependencyControl") local version = DependencyControl{ - name = "ASSFoundation", - version = "0.1.1", - description = "General purpose ASS processing library", - author = "line0", - url = "http://github.com/TypesettingTools/ASSFoundation", - moduleName = "l0.ASSFoundation", - feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json", - { - "l0.ASSFoundation.ClassFactory", - "aegisub.re", "aegisub.util", "aegisub.unicode", - {"l0.ASSFoundation.Common", version="0.1.1", url="https://github.com/TypesettingTools/ASSFoundation", - feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"}, - {"a-mo.LineCollection", version="1.0.1", url="https://github.com/TypesettingTools/Aegisub-Motion"}, - {"a-mo.Line", version="1.0.0", url="https://github.com/TypesettingTools/Aegisub-Motion"}, - {"a-mo.Log", url="https://github.com/TypesettingTools/Aegisub-Motion"}, - "ASSInspector.Inspector", - {"YUtils", optional=true}, + name = "ASSFoundation", + version = "0.1.1", + description = "General purpose ASS processing library", + author = "line0", + url = "http://github.com/TypesettingTools/ASSFoundation", + moduleName = "l0.ASSFoundation", + feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json", + { + "l0.ASSFoundation.ClassFactory", + "aegisub.re", "aegisub.util", "aegisub.unicode", + {"l0.ASSFoundation.Common", version="0.1.1", url="https://github.com/TypesettingTools/ASSFoundation", + feed = "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json"}, + {"a-mo.LineCollection", version="1.0.1", url="https://github.com/TypesettingTools/Aegisub-Motion"}, + {"a-mo.Line", version="1.0.0", url="https://github.com/TypesettingTools/Aegisub-Motion"}, + {"a-mo.Log", url="https://github.com/TypesettingTools/Aegisub-Motion"}, + "ASSInspector.Inspector", + {"YUtils", optional=true}, } local createASSClass, re, util, unicode, Common, LineCollection, Line, Log, ASSInspector, YUtils = version:requireModules() - ``` -A reference to the version record must be added as the *.version* field of your returned module for version control to work. -A module should also register itself to enable circular dependency support. The *:register()* method returns your module, so the last lines of your module should look like this: +A reference to the version record must be added as the _.version_ field of your returned module for version control to work. +A module should also register itself to enable circular dependency support. The _:register()_ method returns your module, so the last lines of your module should look like this: ```lua - MyModule.version = version - return version:register(MyModule) +``` +#### Providing module aliases + +A module may declare additional names it can satisfy via a `provides` field. Once DependencyControl is loaded, any `require` for one of those names — including a bare, non-namespaced name — resolves to your module, _unless_ a real module of that name is already available (yours is only a fallback). This lets a library stand in for a commonly-required dependency without every consuming script having to know your module's namespace. + +```lua +local version = DependencyControl{ + name = "dkjson", + version = "2.10.0", + moduleName = "l0.dkjson", + -- this module can satisfy `require("json")`: + provides = {"json"}, +} ``` ---------------------------------------------- -### Namespaces and Paths ### +Notes: + +- Each entry is a name string (or a table `{name = "json"}`, which may offer further customization options in the future). +- Provided names may be bare/non-namespaced even though your own `moduleName` must be a valid + (dotted) namespace. +- Resolution only applies after DependencyControl itself has been loaded, and always defers to a + genuinely installed module of that name — so users can still bring their own. + +##### Satisfying a dependency with a provider + +`provides` also works across the dependency graph at install time. When a script's `requiredModules` names a module that no feed offers directly, DependencyControl can install a module that lists that name in its `provides` instead — much like Debian's `Provides:` or Arch's `provides`. For example, a script that requires `json` can be satisfied by installing `l0.dkjson`, which provides it. A candidate must actually be installable (its `platforms` must include yours and its version must meet the requirement), a directly-named module is always preferred over a provider, and the usual [package-source precedence](#how-dependencycontrol-selects-package-sources) decides which feed it comes from. Once a provider is installed for a requirement, DependencyControl keeps using and updating that same module rather than switching to a different provider, as long as it can still satisfy the requirement. -DependencyControl strictly enforces a **namespace-based file structure** for modules as well as automation macros in order to ensure there are no conflicts between scripts that happen to have the same name. +A provider can pin _which_ versions of an aliased module it stands in for by giving the table entry an npm-style version range via `provides = {{name = "json", version = "~1.2"}}`. This lets one provider cover a span of releases without being re-published for every patch bump of the aliased module. If not specified, the provider is assumed to satisfy any version requirement for that name. -Automation scripts must define their namespace in the version record whereas for modules the module name (as you would use in a `require` statement) defines the namespace. +For module authors: the `provides` you declare in your version record is mirrored into your feed automatically when you run the `update-feed` CLI, so a published feed advertises it without manual upkeep (you may also add it to a feed by hand). -#### Rules for a valid namespace: #### +#### Advanced: moving a package to a new feed URL - 1. contains _at least_ one dot - 2. must **not** start or end with a dot - 3. must **not** contain series of two or more dots - 4. the character set is restricted to: `A-Z`, `a-z`, `0-9`, `.`, `_`, `-` - 5. *should* be descriptive (this is more of a guideline) +A package can change the feed it updates from by shipping a release whose record points `feed` at the new URL; DependencyControl picks that up the next time it updates the package. Because source selection is trust-aware (see [How DependencyControl Selects Package Sources](#how-dependencycontrol-selects-package-sources)), +plan migrations with that in mind: -__Examples__: - * l0.ASSFoundation - * l0.ASSFoundation.Common (for a separately version-controlled 'submodule') - * l0.ASSWipe - * a-mo.LineCollection +- If the new URL is **already trusted** (listed in DependencyControl's feed, or added by users), the move is seamless. +- If the new URL is **not yet trusted**, automatic updates to it are held back. To avoid breaking them, keep serving from your **old, already-trusted feed in parallel** during the transition and/or get the new URL added to DependencyControl's trusted list. Individual users can also add it to their own `trustedFeeds`. -#### File and Folder Structure #### +--- -The namespace of your script translates into a subtree of the **user**automation directory you can use to store your files in. DepedencyControl will _not_ refuse to work with scripts that ignore this restriction, however it's designed in such a way that downloading to locations outside of your tree is **impossible** (which means your macro/module be able to use the auto-updater). +## Namespaces and Paths -__Automation Scripts__ use the `?user/automation/autoload`, which has a flat file structure. You may **not** use subdirectories and your **file names must start with the namespace of your script**. +DependencyControl strictly enforces a **namespace-based file structure** for modules as well as automation macros in order to ensure there are no conflicts between scripts that happen to have the same name. + +Automation scripts must define their namespace in the version record whereas for modules the module name (as you would use in a `require` statement) defines the namespace. + +Rules for a valid namespace: + +1. contains _at least_ one dot +2. must **not** start or end with a dot +3. must **not** contain series of two or more dots +4. the character set is restricted to: `A-Z`, `a-z`, `0-9`, `.`, `_`, `-` +5. _should_ be descriptive (this is more of a guideline) + +**Examples**: + +- `l0.ASSFoundation` +- `l0.ASSFoundation.Common` (for a separately version-controlled 'submodule') +- `l0.ASSWipe` +- `a-mo.LineCollection` + +### File and Folder Structure + +The namespace of your script translates into a subtree of the **user** automation directory you can use to store your files in: + +- On Windows: `%AppData%\Aegisub\automation` +- On Linux: `~/.aegisub/automation` +- On OSX: `~/Library/Application Support/Aegisub/automation` + +DependencyControl will _not outright_ refuse to work with scripts that ignore this restriction, however it's designed in such a way that downloading to locations outside of your tree is **impossible** (which means your package won't be able to use the auto-updater). + +**Automation Scripts** use the `?user/automation/autoload` directory, which has a flat file structure. You may **not** use subdirectories and your **file names must start with the namespace of your script**. Examples: - * l0.ASSWipe.lua - * l0.ASSWipe.Addon.moon -__Modules__ use the `?user/automation/include` folder, which has a nested file structure. To determine your _subdirectory/file base name_, the dots in your namespace are replaced with `/` (`\` in Windows terms). +- `l0.ASSWipe.lua` +- `l0.ASSWipe.Addon.moon` + +**Modules** use the `?user/automation/include` folder, which has a nested file structure. To determine the base name for your main entry point file and sub-directory, the dots in your namespace are replaced with the path separator (`\` on Windows, `/` on other platforms). -__Tests__ use the `?user/automation/tests/DepUnit/modules` or `?user/automation/tests/DepUnit/macros` folder depending on whether a macro or automation is being tested and mirror the directory structure of the respective `include` and `autoload` folders. +**Tests** live under `?user/automation/tests/DepUnit/modules` or `?user/automation/tests/DepUnit/macros`, depending on whether a module or an automation script is being tested. Test files are resolved as `require` identifiers (`DepUnit.modules.` / `DepUnit.macros.`), so the dots in the namespace always become path separators — the tree is **nested** for both modules and macros, even though the `autoload` folder itself is flat. A test suite is a single file at the root of the namespace path; a multi-file suite adds sibling files under a folder of the same name. -Our example module ASSFoundation with namespace __l0.ASSFoundation__ writes (among others) the following files: - * __?user/automation/include/l0/ASSFoundation__.lua - * __?user/automation/include/l0/ASSFoundation__/ClassFactory.lua - * __?user/automation/include/l0/ASSFoundation__/Draw/Bezier.lua - * __?user/automation/tests/modules/l0/ASSFoundation__.lua +Our example module _ASSFoundation_ with namespace `l0.ASSFoundation` writes (among others) the following files: ---------------------------------------------- +- `?user/automation/include/l0/ASSFoundation.lua` +- `?user/automation/include/l0/ASSFoundation/ClassFactory.lua` +- `?user/automation/include/l0/ASSFoundation/Draw/Bezier.lua` +- `?user/automation/tests/DepUnit/modules/l0/ASSFoundation.lua` -### The Anatomy of an Updater Feed ### +An automation script _MyScript_ with namespace `l0.MyScript` lives at `?user/automation/autoload/l0.MyScript.lua` (flat) but its test suite is nested at `?user/automation/tests/DepUnit/macros/l0/MyScript.lua`. -If you want DepedencyControl auto-update your script on the user's system, you'll need to supply update information in an updater feed, which is a _JSON_ file with a simple basic layout: +--- -*(`//` denotes a comment explaining the property above)* +## The Updater Feed -`````javascript +If you want DependencyControl auto-update your package(s) on the user's system, you'll need to supply update information in an updater feed, which is a _JSON_ file with the following layout: + +_(`//` denotes a comment explaining the property above)_ + +```json { - "dependencyControlFeedFormatVersion": "0.3.0", + "dependencyControlFeedFormatVersion": "0.3.0", // The version of the feed format. The current version is 0.3.0, don't touch this until further notice. "name": "line0's Aegisub Scripts", "description": "Main repository for all of line0's automation macros.", @@ -246,332 +346,440 @@ If you want DepedencyControl auto-update your script on the user's system, you'l "a-mo": "https://raw.githubusercontent.com/TypesettingTools/Aegisub-Motion/DepCtrl/DependencyControl.json", "ASSFoundation": "https://raw.githubusercontent.com/TypesettingTools/ASSFoundation/master/DependencyControl.json" }, - // A hashtable of known feed URLs. Can be referenced with @{feed:name} and will be used to discover other repositories the user can install automation scripts and modules from. At the very least this should contain the repo URLs for the required modules in your repo, but may be used to advertise other unrelated repos you trust. + // A hash table of known feed URLs. Can be referenced with @{feed:name} and will be used to discover other repositories the user can install automation scripts and modules from. At the very least this should contain the repo URLs for the required modules in your repo, but may be used to advertise other unrelated repos you trust. "baseUrl": "https://github.com/TypesettingTools/line0-Aegisub-Scripts", // baseUrl is a template variable that can be referenced in other string fields of the template. It's useful when you have several scripts which all have their documentation hosted on the same site (so they start with the same URL). For more Information about templates, see the section below. "url": "@{baseUrl}", // The address where information about this repository can be found. In this case it references the baseUrl template variable and expands to "https://github.com/TypesettingTools/line0-Aegisub-Scripts". "fileBaseUrl": "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/@{channel}/@{namespace}", // A special rolling template variable. See the templates section below for more information. - + "macros": { // the section where all automation scripts tracked by this feed go. The key for each value is the namespace of the respective script. Below this level, this namespace is available as the @{namespace} and @{namespacePath} template variable - "l0.ASSWipe": { ... }, - "l0.Nudge": { ... } + "l0.ASSWipe": { /* ... */ }, + "l0.Nudge": { /* ... */ } }, "modules": { - // Your modules go here. If your feed doesn't track any modules, you may omit this section (same goes for the macros object) - "l0.ASSFoundation": { ... } - } - -````` + // Your modules go here. If your feed doesn't track any modules, you may omit this section (same goes for the macros object) + "l0.ASSFoundation": { /* ... */ } + } +``` An automation script or module object looks like this: -````javascript +```json "l0.ASSWipe": { - "url": "@{baseUrl}#@{namespace}", - "author": "line0", - "name": "ASSWipe", - "description": "Performs script cleanup, removes unnecessary tags and lines.", - // These script information fields should be identical to the values defined in your - // DepedencyControl version record. - "channels": { - // a list of update channels available for your script (think release, beta and alpha). - // The key is a channel name of your choice, but should make sense to the user picking one. - "master": { - // This example only defines one channel, which is set up to track - // the HEAD of a GitHub repository. - "version": "0.1.3", - // The current script version served in this channel. - // Must be identical to the one in the version record. - "released": "2015-02-26", - // Release date of the current script version (UTC/ISO 8601 format) - "default": true, - // Marks this channel as the default channel in case the user doesn't have picked a specific one. - // Must be set to true for **exactly** one channel in the list. - "platforms": ["Windows-x86", "Windows-x64", "OSX-x64"] - // Optional: A list of platforms you serve builds for. You should omit this property for regular scripts - // and modules that use only Lua/Moonscript and no binaries. If this property is absent, - // the platform check will be skipped. The platform names are derived from the output of - // ffi.os()-ffi.arch() in luajit. - "files": [ - // A list of files installed by your script. - { - "name": ".lua", - // the file name relative to the path assigned to the script by your namespace choice - // (see 3. Namespaces and Paths for more information). Available as the @{fileName} template variable - // for use in the url field below. - "url": "@{fileBaseUrl}@{fileName}", - // URL from which the **raw** file can be downloaded from (no archives, no javascript - // redirects, etc...). In this case the templates expand to - // "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/l0.ASSWipe.lua" - "sha1": "A7BD1C7F0E776BA3010B1448F22DE6528F73B077" - // The SHA-1 hash of the file being currently served under that url. Will be checked - // against the downloaded file, so it must always be present and valid or the update process - // will fail on the user's end. - }, - { - "name": ".lua", - "type": "test", - // Optional, defaults to "script". Specify "test" to denote a unit test. - // Currently only "script" and "test" are available, unknown script types will be skipped. - "url": "@{fileBaseUrl}.Tests.lua", - "sha1": "27745AB9CF04A840CF3454050CA9D38FA345CEBB" - }, - { - "name": ".Helper.dll", - "url": "@{fileBaseUrl}@{fileName}", - "sha1": "0B4E0511116355D4A11C2EC75DF7EEAD0E14DE9F" - "platform": "Windows-x86" - // Optional. When this property is present, the file will only be downloaded to the users - // computer if his platform matches to this value. - } - ], - "requiredModules": [ - // an exhaustive list of modules required by this script. Must be identical to the required - // module entries in your DepdencyControl record, but you may not use short style here. - // (see 2. Usage for Automation Scripts for more information) - { - "moduleName": "a-mo.LineCollection", - "name": "Aegisub-Motion (LineCollection)", - "url": "https://github.com/torque/Aegisub-Motion", - "version": "1.0.1", - "feed": "@{feed:a-mo}" - }, - { - "moduleName": "l0.ASSFoundation", - "name": "ASSFoundation", - "url": "https://github.com/TypesettingTools/ASSFoundation", - "version": "0.1.1", - "feed": "@{feed:ASSFoundation}" - }, - { - "moduleName": "aegisub.util" - }, - ] + "url": "@{baseUrl}#@{namespace}", + "author": "line0", + "name": "ASSWipe", + "description": "Performs script cleanup, removes unnecessary tags and lines.", + // These script information fields should be identical to the values defined in your + // DependencyControl version record. + "channels": { + // a list of update channels available for your script (think release, beta and alpha). + // The key is a channel name of your choice, but should make sense to the user picking one. + "master": { + // This example only defines one channel, which is set up to track + // the HEAD of a GitHub repository. + "version": "0.1.3", + // The current script version served in this channel. + // Must be identical to the one in the version record. + "released": "2015-02-26", + // Release date of the current script version (UTC/ISO 8601 format) + "default": true, + // Marks this channel as the default channel in case the user doesn't have picked a specific one. + // Must be set to true for **exactly** one channel in the list. + "platforms": ["Windows-x86", "Windows-x64", "OSX-x64"] + // Optional: A list of platforms you serve builds for. You should omit this property for regular scripts + // and modules that use only Lua/Moonscript and no binaries. If this property is absent, + // the platform check will be skipped. The platform names are derived from the output of + // ffi.os()-ffi.arch() in luajit. + "files": [ + // A list of files installed by your script. + { + "name": ".lua", + // the file name relative to the path assigned to the script by your namespace choice + // (see 3. Namespaces and Paths for more information). Available as the @{fileName} template variable + // for use in the url field below. + "url": "@{fileBaseUrl}@{fileName}", + // URL from which the **raw** file can be downloaded from (no archives, no javascript + // redirects, etc...). In this case the templates expand to + // "https://raw.githubusercontent.com/TypesettingTools/line0-Aegisub-Scripts/master/l0.ASSWipe.lua" + "sha1": "A7BD1C7F0E776BA3010B1448F22DE6528F73B077" + // The SHA-1 hash of the file being currently served under that url. Will be checked + // against the downloaded file, so it must always be present and valid or the update process + // will fail on the user's end. + }, + { + "name": ".lua", + "type": "test", + // Optional, defaults to "script". Specify "test" to denote a unit test. + // Currently only "script" and "test" are available, unknown script types will be skipped. + "url": "@{fileBaseUrl}.Tests.lua", + "sha1": "27745AB9CF04A840CF3454050CA9D38FA345CEBB" + }, + { + "name": ".Helper.dll", + "url": "@{fileBaseUrl}@{fileName}", + "sha1": "0B4E0511116355D4A11C2EC75DF7EEAD0E14DE9F", + "platform": "Windows-x86" + // Optional. When this property is present, the file will only be downloaded to the users + // computer if his platform matches to this value. } - }, - "changelog": { - // a change log that allows users to see what's new in this and previous versions. The changelog - // is shared between all channels. Only the entries with a version number equal or below - // the version the user just updated to will be displayed. - "0.1.0": [ - "Sync with ASSFoundation changes", - // one entry for each line - "Start versioning with DependencyControl" - ], - "0.1.3": [ - "Enabled auto-update using DependencyControl", - "Changed config file to \\config\\l0.ASSWipe.json (rename ASSWipe.json to restore your existing configuration)", - "DependencyControl compatibility fixes" - ] - } + ], + "requiredModules": [ + // an exhaustive list of modules required by this script. Must be identical to the required + // module entries in your DependencyControl record, but you may not use short style here. + // (see 2. Usage for Automation Scripts for more information) + { + "moduleName": "a-mo.LineCollection", + "name": "Aegisub-Motion (LineCollection)", + "url": "https://github.com/torque/Aegisub-Motion", + "version": "1.0.1", + "feed": "@{feed:a-mo}" + }, + { + "moduleName": "l0.ASSFoundation", + "name": "ASSFoundation", + "url": "https://github.com/TypesettingTools/ASSFoundation", + "version": "0.1.1", + "feed": "@{feed:ASSFoundation}" + }, + { + "moduleName": "aegisub.util" + }, + ] } -```` + }, + "changelog": { + // a change log that allows users to see what's new in this and previous versions. The changelog + // is shared between all channels. Only the entries with a version number equal or below + // the version the user just updated to will be displayed. + "0.1.0": [ + "Sync with ASSFoundation changes", + // one entry for each line + "Start versioning with DependencyControl" + ], + "0.1.3": [ + "Enabled auto-update using DependencyControl", + "Changed config file to \\config\\l0.ASSWipe.json (rename ASSWipe.json to restore your existing configuration)", + "DependencyControl compatibility fixes" + ] + } +} +``` + +Full _JSON Schema_ documents (which you can use to validate your feeds) are provided for the following feed versions: -#### Template Variables #### +- [v0.3.0](./schemas/feed/v0.3.0.json) (also validates legacy _v0.2.0_ feeds) -To make maintaining an update feed easier, you can use several template variables that will be expanded when used inside string values (but **not** Keys). +### Template Variables -__Regular Variables:__ These reference a specific key or value and are available at the same depth and further down the tree from the point on where they were created. +To make maintaining an update feed easier, you can use several template variables that will be expanded when used inside string values (but **not** keys). + +**Regular Variables**: These reference a specific key or value and are available at the same depth and further down the tree from the point on where they were created. Variables extracted at the **same depth** are expanded in a specific order. As a consequence only references to variables of lower order are expanded in values that are assigned to a variable themselves. _Depth 1:_ Feed Information - 1. __feedName__: The name of the feed - 2. __baseUrl__: The baseUrl field - 3. __feed:###__: A reference to a feed URL in the knownFeeds table + +1. `@{feedName}`: The name of the feed +2. `@{baseUrl}`: The baseUrl field +3. `@{feed:}`: A reference to a feed URL in the knownFeeds table _Depth 3:_ Script Information - 1. __namespace__: the script namespace - 2. __namespacePath__: the script namespace with all `.` replaced by `/` - 3. scriptName: the script name + +1. `@{namespace}`: the script namespace +2. `@{namespacePath}`: the script namespace with all `.` replaced by `/` +3. `@{scriptName}`: the script name _Depth 5:_ Version Information - 1. __channel__: the channel name of this version record - 2. __version__: the version number as a SemVer string -_Depth 7:_ File Information - 1. __platform__: the platform defined for this file, otherwise an empty string - 2. __fileName__: the file name +1. `@{channel}`: the channel name of this version record +2. `@{version}`: the version number as a SemVer string +_Depth 7:_ File Information -__"Rolling" Variables:__ These variables can be defined at any depth in the JSON tree and are continuously expanded using the variables available. You can reference a rolling variable in itself, which will substitute the template for the contents the variable had at the parent-level. +1. `@{platform}`: the platform defined for this file, otherwise an empty string +2. `@{fileName}`: the file name -Right now there's only one such variable: __fileBaseUrl__, which you can use to construct the URL to a file using the template variables available. +**"Rolling" Variables**: These variables can be defined at any depth in the JSON tree and are continuously expanded using the variables available. You can reference a rolling variable in itself, which will substitute the template for the contents the variable had at the parent-level. -For an example to serve updates from the HEAD of a GitHub repository, see [here](https://github.com/TypesettingTools/line0-Aegisub-Scripts/blob/master/DependencyControl.json). An example that shows a feed making use of tagged releases is [also available](https://github.com/TypesettingTools/ASSFoundation/blob/master/DependencyControl.json). +Right now there's only one such variable: `@{fileBaseUrl}`, which you can use to construct the URL to a file using the template variables available. - --------------------------------------------- +For an example to serve updates from the HEAD of a GitHub repository main branch, see [here](https://github.com/TypesettingTools/line0-Aegisub-Scripts/blob/master/DependencyControl.json). An example that shows a feed making use of tagged releases is [also available](https://github.com/TypesettingTools/ASSFoundation/blob/master/DependencyControl.json). -### Reference ### +## Reference This section is currently both incomplete and outdated. Sorry about that. -#### DependencyControl #### +### DependencyControl -__DependencyControl{*tbl* [requiredModules]={}, *str* :name=script_name, *str* :description=script_description, *str* :author=script_author, *str* :url, *str* :version, *str* :moduleName, *str* [:configFile], *string* [:namespace]} --> *obj* DependecyControlRecord__ +**DependencyControl{_tbl_ [requiredModules]={}, _str_ :name=script*name, \_str* :description=script*description, \_str* :author=script*author, \_str* :url, _str_ :version, _str_ :moduleName, _str_ [:configFile], _string_ [:namespace]} --> _obj_ DependencyControlRecord** -The constructor for a DepedencyControl record. Uses the table-based signature. -__Arguments:__ +The constructor for a DependencyControl record. Uses the table-based signature. +**Arguments:** - * _requiredModules_: the first and only unnamed argument. Contains all required modules, which may be either a single string for a non-version-controlled requirement or a table with the following fields: - * __*str* [moduleName/[1]]:__ the module name - * __*str* [version]:__ The minimum required version of the module. Must conform to Semantic Versioning standards. The module in question must contain a DependencyControl version record or otherwise compatible version number. - * __*str* [url]__: The URL of the site where the module can be downloaded from (will be shown to the user in error methods). - * __*str* [feed]__: The update feed used to fetch a copy of the required module when it is missing from the user's system. - * __*bool* [optional=false]__: Marks the module as an optional requirement. If the module is missing on the user's system, no error will be thrown. However, version requirements *will* be checked if the module was found. - * __*str* [name]__: Friendly module name (used for error messages). +- _requiredModules_: the first and only unnamed argument. Contains all required modules, which may be either a single string for a non-version-controlled requirement or a table with the following fields: + - **_str_ [moduleName/[1]]:** the module name + - **_str_ [version]:** The minimum required version of the module. Must conform to Semantic Versioning standards. The module in question must contain a DependencyControl version record or otherwise compatible version number. + - **_str_ [url]**: The URL of the site where the module can be downloaded from (will be shown to the user in error methods). + - **_str_ [feed]**: The update feed used to fetch a copy of the required module when it is missing from the user's system. + - **_bool_ [optional=false]**: Marks the module as an optional requirement. If the module is missing on the user's system, no error will be thrown. However, version requirements _will_ be checked if the module was found. + - **_str_ [name]**: Friendly module name (used for error messages). -* _name, description, author_: Required for modules, pulled from the *script_* globals for macros. -* _version_: Must conform to [Semantic Versioning](http://semver.org/) standards. Labels and build metadata are not supported at this time -* _moduleName_: module name (as used in require statements). Required for modules, must be nil for macros. Represents the namespace of a module. -* _url_: The web site/repository URL of your script -* _feed_: The update feed for your script. -* _configFile_: Configuration file base name used by the script. Defaults to the namespace. Used for configuration services and script management purposes. +- _name, description, author_: Required for modules, pulled from the \_script\_\_ globals for macros. +- _version_: Must conform to [Semantic Versioning](http://semver.org/) standards. Labels and build metadata are not supported at this time +- _moduleName_: module name (as used in require statements). Required for modules, must be nil for macros. Represents the namespace of a module. +- _url_: The web site/repository URL of your script +- _feed_: The update feed for your script. +- _configFile_: Configuration file base name used by the script. Defaults to the namespace. Used for configuration services and script management purposes. -##### Methods ##### -__:checkVersion(*str/num* version, *str* [precision = "patch"]) --> *bool* moduleUpToDate, *str* error__ +#### Methods -Returns true if the version number of the record is greater than or equal to __version__. Reduce the __precision__ to `minor` or `major` to also return true for lower patch or minor versions respectively. If the version can't be parsed it returns nil and and error message. +**:checkVersion(_str/num_ version, _str_ [precision = "patch"]) --> _bool_ moduleUpToDate, _str_ error** -__:checkOptionalModules(*tbl* modules) --> *bool* result, *str* errorMessage__ +Returns true if the version number of the record is greater than or equal to **version**. Reduce the **precision** to `minor` or `major` to also return true for lower patch or minor versions respectively. If the version can't be parsed it returns nil and and error message. -Returns true if the optional __modules__ have been loaded, where __modules__ is a list of module names. If one or more of the modules are missing it returns false and an error message. +**:checkOptionalModules(_tbl_ modules) --> _bool_ result, _str_ errorMessage** -__:getConfigFileName() --> *str* fileName__ +Returns true if the optional **modules** have been loaded, where **modules** is a list of module names. If one or more of the modules are missing it returns false and an error message. + +**:getConfigFileName() --> _str_ fileName** Returns a full path to the config file proposed for this script by DependencyControl. Uses the configFile argument passed to the constructor which defaults to the script namespace. The path is subject to user configuration and defaults to "?user\config". The file ending is always .json, because why would you use any other format? -The rationale for this function is to keep all macro and module configuration files neatly in one spot and make them discoverable for other scripts (through the DepedencyControl config file). +The rationale for this function is to keep all macro and module configuration files neatly in one spot and make them discoverable for other scripts (through the DependencyControl config file). -__:getConfigHandler([defaults], [section], [noLoad]) => *obj* ConfigHandler__ +**:getConfigHandler([defaults], [section], [noLoad]) => _obj_ ConfigHandler** Returns a ConfigHandler (see [ConfigHandler Documentation](#FIXME)) attached to the config file configured for this script. -__:getLogger(*tbl* args) => *obj* Logger__ +**:getLogger(_tbl_ args) => _obj_ Logger** Returns a Logger (see [Logger Documentation](#FIXME)) preconfigured for this script. Trace level and config file preference default to user-configurable values. Log file name and prefix are based on namespace and script name. -__:getVersionNumber(*str/num* versionString) --> *int/bool* version, *str* error__ +**:getVersionNumber(_str/num_ versionString) --> _int/bool_ version, _str_ error** Takes a SemVer string and converts it into a version number. If parsing the version string fails it returns false and an error message instead. -__:getVersionString(*int* [version=@version]) --> *str* versionString__ +**:getVersionString(_int_ [version=@version]) --> _str_ versionString** Returns a version (by default the script version) as a SemVer string. -__:getConfigFileName() --> *str* configFileName__ +**:getConfigFileName() --> _str_ configFileName** Generates and returns a full path to the registered config file name for the module. -__:loadConfig(*bool* [importRecord], *bool* [forceReloadGlobal]) --> *bool* shouldWriteConfig, *bool* firstInit__ +**:loadConfig(_bool_ [importRecord], _bool_ [forceReloadGlobal]) --> _bool_ shouldWriteConfig, _bool_ firstInit** -Loads global DependencyControl and per-script configuration from the DepedencyControl configuration file. If __importRecord__ is true, the version record information of a DependencyControl record will be (temporarily) overwritten by the values contained in the configuration file. -Global configuration is only loaded on first run or if __forceReloadGlobal__ is true. +Loads global DependencyControl and per-script configuration from the DependencyControl configuration file. If **importRecord** is true, the version record information of a DependencyControl record will be (temporarily) overwritten by the values contained in the configuration file. +Global configuration is only loaded on first run or if **forceReloadGlobal** is true. The first return result indicates there are changes to be written to the config file, the second result returns true if the config file was only just created. _Intended for internal use._ -__:loadModule(*tbl* module, *bool* [usePrivate]) --> *tbl* moduleRef__ +**:loadModule(_tbl_ module, _bool_ [usePrivate]) --> _tbl_ moduleRef** -Loads and returns single module and only errors out in case of module errors. Intended for internal use. If __usePrivate__ is true, a private copy of the module is loaded instead. +Loads and returns single module and only errors out in case of module errors. Intended for internal use. If **usePrivate** is true, a private copy of the module is loaded instead. -__:moveFile(*str* src, *str* dest) --> *bool* success, *str* error__ +**:moveFile(_str_ src, _str_ dest) --> _bool_ success, _str_ error** -Moves a file from __source__ to __destiantion__ (where both are full file names). Returns true on success or false and error message on failure. +Moves a file from **source** to **destination** (where both are full file names). Returns true on success or false and error message on failure. -__:register(*tbl* selfRef, extraUnitTestArgs...) --> *tbl* selfRef__ +**:register(_tbl_ selfRef, extraUnitTestArgs...) --> _tbl_ selfRef** Replaces dummy reference written to the global LOADED_MODULES table at DependencyControl object creation time with a reference to this module. -Also automatically registers unit tests for this module, passing in any __extraUnitTestArgs__ +Also automatically registers unit tests for this module, passing in any **extraUnitTestArgs** The purpose of this construct is to allow circular references between modules. Limitations apply: the modules in question may not use each other during construction/setup of each module (for obvious reasons). Call this method as replacement for returning your module. -__:registerMacro(*str* [name=@name], *str* [description=@description], *func* processing_function, *func* [validation_function], *func* is_active_function, *bool|string* [submenu=false])__ +**:registerMacro(_str_ [name=@name], _str_ [description=@description], _func_ processing*function, \_func* [validation_function], _func_ is*active_function, \_bool|string* [submenu=false])** Alternative Signature: -__:registerMacro(*func* processing_function, *func* [validation_function], *func* is_active_function, *bool|string* [submenu=false])__ +**:registerMacro(_func_ processing*function, \_func* [validation_function], _func_ is*active_function, \_bool|string* [submenu=false])** Registers a single macro using script name and description by default. -Use __submenu__ to specify a submenu name to use for this macro or set it to `true` to use the automation script name. +Use **submenu** to specify a submenu name to use for this macro or set it to `true` to use the automation script name. -If the script entry in the DependencyControl configuration file contains a __customMenu__ property, the macro will be placed in the specified menu. Do note that that this setting is for *user customization* and not to be changed without the user's consent. +If the script entry in the DependencyControl configuration file contains a **customMenu** property, the macro will be placed in the specified menu. Do note that that this setting is for _user customization_ and not to be changed without the user's consent. For the other arguments, please refer to the [aegisub.register_macro](http://docs.aegisub.org/latest/Automation/Lua/Registration/#aegisub.register_macro) API documentation. -__:registerMacros(*tbl* macros, *bool|string* [submenuDefault=true])__ +**:registerMacros(_tbl_ macros, _bool|string_ [submenuDefault=true], _tbl_ [testExports])** + +Registers multiple macros, where **macros** is a list of tables containing the arguments to a **:registerMacro()** call for each automation menu entry. +Use **submenuDefault** to specify a submenu all macros will be placed in unless overridden on a per-macro basis. Defaults to `true` which causes the automation script name to be used as the submenu name. +Pass **testExports** to expose the script's internal values (helpers, etc.) to its unit test suite; they are forwarded to the suite's import function so an automation script can be unit-tested (see **:registerTests**). -Registers multiple macros, where __macros__ is a list of tables containing the arguments to a __:registerMacro()__ call for each automation menu entry. a single macro using script name and description by default. -Use __submenuDefault__ to specify a submenu all macros will be placed in unless overriden on a per-macro basis. Defaults to `true` which causes the automation script name to be used as the submenu name. +**:registerTests(unitTestArgs...)** -__:registerTests(unitTestArgs...)__ +Loads and registers this script's or module's DependencyControl unit test suite when one is present, forwarding any **unitTestArgs** to the suite's import function. You rarely call it directly: modules register their tests automatically through **:register**, and automation scripts through **:registerMacro**/**:registerMacros** (an automation script passes its internals via the **testExports** argument of **:registerMacros**). -Registers unit tests for automation modules, passing in any of specified __unitTestArgs__. Registration of modules is done automatically upon calling __:register__ +The suite's import function is called with, in order: the subject under test (for a module its own reference; for an automation script a map of its registered macros keyed by name, each carrying the macro's unhooked `process`/`validate`/`isActive`), the script's dependencies, any extra arguments (a module's own table, or an automation script's `testExports`), and a controls object as the final argument. -__:requireModules([modules=@requiredModules], *bool* [forceUpdate], *bool* [updateMode], *tbl* [addFeeds={@feed})] --> ...__ +**:requireModules([modules=@requiredModules], _bool_ [forceUpdate], _bool_ [updateMode], _tbl_ [addFeeds={@feed})] --> ...** Loads the modules required by this script and returns a reference for every requirement in the order they were supplied by the user. If an optional module is not found, nil is returned. -The updater will try to download copies of modules that are missing or outdated on the user's system. The __addFeeds__ parameter can be used to supply additional feeds to search. If missing/outdated requirements can't be fetched, the method will throw an error in normal mode or false and an error message in __update mode__. +The updater will try to download copies of modules that are missing or outdated on the user's system. The **addFeeds** parameter can be used to supply additional feeds to search. If missing/outdated requirements can't be fetched, the method will throw an error in normal mode or false and an error message in **update mode**. -Use __forceUpdate__ to override update intervals and perform update checks for all required modules, even if requirements are satisfied. +Use **forceUpdate** to override update intervals and perform update checks for all required modules, even if requirements are satisfied. -__:writeConfig(*bool* [writeLocal=true], *bool* [writeGlobal=true], *bool* [concert]]__ +**:writeConfig(_bool_ [writeLocal=true], _bool_ [writeGlobal=true], _bool_ [concert]]** -Writes __global__ and per-module __local__ configuration. If __concert__ is true, concerted writing will be used to update the configuration of all DependencyControl hosted by any given macro/environment at once. See ConfigHandler documentation for more information. _Intended for internal use._ +Writes **global** and per-module **local** configuration. If **concert** is true, concerted writing will be used to update the configuration of all DependencyControl hosted by any given macro/environment at once. See ConfigHandler documentation for more information. _Intended for internal use._ -#### Updater ##### +### Updater -##### Methods ##### +#### Methods -__:getUpdaterErrorMsg(*int* [code], *str* targetName, ...) --> *str* errorMsg__ +**:getUpdaterErrorMsg(_int_ [code], _str_ targetName, ...) --> _str_ errorMsg** -Used to turn an updater return __code__ into a human-readable error message. The __name__ of the updated component and other format string parameters are passed into the function. +Used to turn an updater return **code** into a human-readable error message. The **name** of the updated component and other format string parameters are passed into the function. VarArgs: - 1. __*bool* isModule__: True when component is a module, false when it is an automation script/macro - 2. __*bool* isFetch__: True when we are fetching a missing module, false when updating - 3. __extError__: Extended error information as returned by the _:update()_ method +1. **_bool_ isModule**: True when component is a module, false when it is an automation script/macro +2. **_bool_ isFetch**: True when we are fetching a missing module, false when updating +3. **extError**: Extended error information as returned by the _:update()_ method -__:getUpdaterLock(*bool* [doWait], *int* [waitTimeout=(user config)]) --> *bool* result, *str* runningHost__ +**:getUpdaterLock(_bool_ [doWait], _int_ [waitTimeout=(user config)]) --> _bool_ result, _str_ runningHost** Locks the updater to the current macro/environment. Since all automation scripts load in parallel we have to make sure multiple automation scripts don't all update/fetch the same dependencies at once multiple times. The solution is to only let one updater operate at a time. The others will wait their turn and recheck if their required modules were fetched in the meantime. -If __doWait__ is true, the function will wait until the updater is unlocked or __waitTimeout__ has passed. It will then get the lock and return true. If __doWait__ is false, the function will return immediately (true on success, false if another updater has the lock). _Intendend for internal use_. +If **doWait** is true, the function will wait until the updater is unlocked or **waitTimeout** has passed. It will then get the lock and return true. If **doWait** is false, the function will return immediately (true on success, false if another updater has the lock). _Intended for internal use_. -__:releaseUpdaterLock()__ +**:releaseUpdaterLock()** Makes an updater host (macro) release its lock on the Updater if it has one. See _:getUpdaterLock_ for more information -__:update(*bool* [force], *tbl* [addFeeds], *bool* [tryAllFeeds=auto]) --> *int* resultCode, *str* extError__ +**:update(_bool_ [force], _tbl_ [addFeeds]) --> _int_ resultCode, _str_ extError** -Runs the updater on this automation script or module. This includes recursively updating all required modules. When __force__ is true, required modules will skip their update interval check. +Runs the updater on this automation script or module. This includes recursively updating all required modules. When **force** is true, required modules will skip their update interval check. -By default, the updater will process all suitable feeds until one feed confirms the script to be up-to-date (unless configured otherwise by the user or if we are looking for updates to an outdated component). Set __tryAllFeeds__ to true to check all feeds until an update is found. You can also supply __additional candidate feeds__. +The updater consults feeds in trust order (closest and most-trusted first) and stops as soon as a source can satisfy the requirement — see [How DependencyControl Selects Package Sources](#how-dependencycontrol-selects-package-sources). You can supply **additional candidate feeds**. Returns a result code (0: up-to-date, 1: update performed, <=-1: error) and extended error information which can be fed into _:getUpdaterErrorMsg()_ to get a descriptive error message. -#### Logger #### +### Logger tbd -#### ConfigHandler #### +### ConfigHandler tbd -#### FileOps #### +### FileOps tbd -#### UnitTestSuite #### +### UnitTestSuite -Reference documentation for the UnitTestSuite module is available in the [source code](https://github.com/TypesettingTools/DependencyControl/blob/master/modules/DependencyControl/UnitTestSuite.moon#L760) +Reference documentation for the UnitTestSuite module is available in the [source code](https://github.com/TypesettingTools/DependencyControl/blob/master/modules/l0/DependencyControl/UnitTestSuite.moon#L760) -#### UpdateFeed #### +### UpdateFeed tbd + +## CLI + +DependencyControl ships a CLI launcher (`depctrl.lua`) for running tests, building release +bundles, and deploying to a local Aegisub installation — all **without** a running Aegisub +process. All commands read their package list from a feed JSON file and can operate on any +DepCtrl-managed package, not only DependencyControl itself. + +### Prerequisites + +- _LuaJIT_ on your `PATH`, built with `DLUAJIT_ENABLE_LUA52COMPAT` +- _LuaRocks_, configured for Lua v5.1, which _LuaJIT_ is ABI-compatible with. You may have to select the Lua version explicitly via `luarocks --lua-version=5.1` +- The [moonscript](https://luarocks.org/modules/leafo/moonscript), [LuaFileSystem](https://luarocks.org/modules/hisham/luafilesystem) and [argparse](https://luarocks.org/modules/mpeterv/argparse) rocks, installed into that 5.1 tree: + + ```sh + luarocks --lua-version=5.1 install moonscript + luarocks --lua-version=5.1 install luafilesystem + luarocks --lua-version=5.1 install argparse + ``` + +- Your `LUA_PATH` / `LUA_CPATH` must let `luajit` find the LuaRocks-installed modules (`luarocks --lua-version=5.1 path --bin` prints the correct values). + +General form: + +```sh +luajit depctrl.lua [options] +``` + +The feed is resolved in this order: `--feed` flag → `DependencyControl.json` in the current +working directory. All commands accept `--target-module` and `--target-macro` to restrict +processing to specific packages; without them the command operates on every package in the feed. + +### `test` — Run unit test suites + +```sh +luajit depctrl.lua test [--feed ] [--report-dir ] + [--target-module ] [--target-macro ] +``` + +Loads every matching package from the feed, runs its DepUnit test suite (if one is registered), +and writes a per-package [CTRF](https://ctrf.io) JSON report. Exit code `0` = all tested +packages passed, `1` = one or more failures or load errors. + +Packages without a test suite are skipped with a notice; packages that fail to load are counted +as failures. Log files and config/feed caches are written to a per-run throwaway workspace under +the system temp directory rather than touching your real Aegisub configuration. + +The feed must have correct `localFileBasePath` entries so the CLI can resolve source files on +disk. + +| Option | Default | Description | +| ----------------- | ------------------------------- | -------------------------------------------------------- | +| `--feed` | `DependencyControl.json` in CWD | Path to the feed JSON file | +| `--report-dir` | `ctrf/` | Directory for per-package CTRF JSON reports | +| `--target-module` | _(all modules)_ | Module namespace to test; repeatable | +| `--target-macro` | _(all macros)_ | Macro namespace to test; repeatable | + +### `bundle` — Build a release archive + +```sh +luajit depctrl.lua bundle [--feed ] [--out-dir ] + [--target-module ] [--target-macro ] +``` + +Copies every file listed in the feed into a `dist/` subfolder of ``, then packages +`dist/` into a zip archive named `-v[--g].zip` in ``. +`dist/` is wiped and recreated on each run. The git branch and hash suffix is omitted when +HEAD is exactly on a tag. + +| Option | Default | Description | +| ----------------- | ------------------------------- | -------------------------------------------- | +| `--feed` | `DependencyControl.json` in CWD | Path to the feed JSON file | +| `--out-dir` | CWD | Root for the `dist/` folder and the zip file | +| `--target-module` | _(all modules)_ | Restrict to this module namespace; repeatable | +| `--target-macro` | _(all macros)_ | Restrict to this macro namespace; repeatable | + +Exit code `0` = success, `1` = one or more errors. + +### `deploy` — Deploy to a local Aegisub installation + +```sh +luajit depctrl.lua deploy [--feed ] [--out-dir ] [--clobber | --no-clobber] + [--target-module ] [--target-macro ] +``` + +Copies every file listed in the feed directly into `` using the Aegisub install layout — +macros into `/automation/autoload/`, modules into `/automation/modules/`, test files +into `/automation/tests/DepUnit/…`. Useful for testing against a locally installed Aegisub +without going through a full release build. + +| Option | Default | Description | +| ----------------- | ------------------------------- | ------------------------------------------------------ | +| `--feed` | `DependencyControl.json` in CWD | Path to the feed JSON file | +| `--out-dir` | CWD | Deployment root — typically the Aegisub user directory | +| `--clobber` | false | Overwrite existing files in the deployment directory | +| `--no-clobber` | _(default)_ | Skip files that already exist at the destination | +| `--target-module` | _(all modules)_ | Restrict to this module namespace; repeatable | +| `--target-macro` | _(all macros)_ | Restrict to this macro namespace; repeatable | + +Exit code `0` = success, `1` = one or more errors. diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..e53e20c --- /dev/null +++ b/cspell.json @@ -0,0 +1,35 @@ +{ + "dictionaryDefinitions": [ + { + "name": "lua", + "path": "./.cspell/lua.txt", + "scope": "workspace", + "addWords": true + }, + { + "name": "ffi", + "path": "./.cspell/ffi.txt", + "scope": "workspace", + "addWords": true + }, + { + "name": "domain-specific", + "path": "./.cspell/domain-specific.txt", + "scope": "workspace", + "addWords": true + } + ], + "dictionaries": ["domain-specific"], + "languageSettings": [ + { + "languageId": "lua", + "dictionaries": ["lua", "ffi"] + }, + { + "languageId": "moonscript", + "dictionaries": ["lua", "ffi"] + } + ], + "words": ["moonscript"], + "ignorePaths": [".cspell/**"] +} diff --git a/depctrl.lua b/depctrl.lua new file mode 100644 index 0000000..b84979d --- /dev/null +++ b/depctrl.lua @@ -0,0 +1,465 @@ +#!/usr/bin/env luajit +-- DependencyControl CLI toolbox + +local ffi = require "ffi" +local lfs = require "lfs" +local argparse = require "argparse" +require "moonscript" -- installs moonscript's package.moonpath loader for .moon files + +-- ── Path utilities ──────────────────────────────────────────────────────────── + +local isWindows = ffi.os == "Windows" +local pathSep = isWindows and "\\" or "/" + +local function dirname(path) + return (path or ""):match("^(.*)[/\\][^/\\]*$") or "." +end + +local function isAbsolute(path) + return path:match("^%a:[/\\]") ~= nil -- C:\... + or path:match("^[/\\]") ~= nil -- /... or \... +end + +local function resolveAbsPath(path) + if not isAbsolute(path) then + return lfs.currentdir() .. pathSep .. path + end + return path +end + +-- ── Argument parsing ────────────────────────────────────────────────────────── + +local parser = argparse("depctrl", "DependencyControl CLI toolbox") + :epilog("See README.md for detailed instructions.") +parser:command_target("command") + +-- Selector options shared by all commands: repeat --target-module / --target-macro to pick +-- packages by namespace. With none given, a command operates on every package in the feed. +local function addTargets(cmd) + cmd:option("--target-module", "Module namespace to operate on (repeatable; default: all)") + :argname(""):count("*") + cmd:option("--target-macro", "Macro namespace to operate on (repeatable; default: all)") + :argname(""):count("*") +end + +local testCmd = parser:command("test", "Run the unit test suite(s) for packages in a feed") +testCmd:option("-f --feed", "Feed JSON path"):default("DependencyControl.json") +testCmd:option("-r --report-dir", "Directory for per-package CTRF JSON reports"):default("ctrf") +addTargets(testCmd) + +local bundleCmd = parser:command("bundle", "Build a dist/ release bundle and zip archive") +bundleCmd:option("-f --feed", "Feed JSON path"):default("DependencyControl.json") +bundleCmd:option("-o --out-dir", "Output directory; script files go into its dist/ subfolder"):default(".") +addTargets(bundleCmd) + +local deployCmd = parser:command("deploy", "Deploy files directly to an output directory") +deployCmd:option("-f --feed", "Feed JSON path"):default("DependencyControl.json") +deployCmd:option("-o --out-dir", "Output directory"):default(".") +deployCmd:flag("--clobber", "Overwrite existing files (default)"):target("clobber") +deployCmd:flag("--no-clobber", "Skip files that already exist at the destination"):target("clobber"):action("store_false") +addTargets(deployCmd) + +local validateCmd = parser:command("validate-schema", + "Validate a config or feed JSON file against its DependencyControl JSON schema") +validateCmd:option("-f --file", "JSON file to validate"):argname("") +validateCmd:option("-t --type", "Schema family to validate against: 'config' or 'feed'"):argname("") +validateCmd:option("--schema-version", + "Validate against a specific schema version (e.g. 0.7.0) instead of auto-selecting the best match"):argname("") + +local updateFeedCmd = parser:command("update-feed", + "Refresh SHA-1 hashes, version info, and file presence in a feed channel") +updateFeedCmd:option("-f --feed", "Feed JSON path"):default("DependencyControl.json") +updateFeedCmd:option("-c --channel", "Channel to update (default: the channel marked default: true)") + :argname("") +updateFeedCmd:flag("-n --dry-run", "Print what would change without writing back") +addTargets(updateFeedCmd) + +local args = parser:parse() + +-- ── Resolve the launcher directory ─────────────────────────────────────────── +-- Made absolute up-front so nothing downstream can be confused by CWD changes. + +local launcherDir = dirname(arg and arg[0]) +if launcherDir == "." then + launcherDir = lfs.currentdir() +elseif not isAbsolute(launcherDir) then + launcherDir = lfs.currentdir() .. pathSep .. launcherDir +end + +-- ── Module resolution ───────────────────────────────────────────────────────── +-- The repo's modules/ tree is namespaced (modules/l0/…), so l0.* require paths map +-- straight onto it: moonscript's loader resolves .moon via package.moonpath, the +-- stock searcher the vendored .lua via package.path. No custom searcher needed. + +local depCtrlModulesDir = launcherDir .. pathSep .. "modules" +package.path = ("%s/?.lua;%s/?/init.lua;"):format(depCtrlModulesDir, depCtrlModulesDir) .. package.path +package.moonpath = ("%s/?.moon;%s/?/init.moon;"):format(depCtrlModulesDir, depCtrlModulesDir) .. (package.moonpath or "") + +if isWindows then + require("l0.DependencyControl.helpers.ffi-windows").setConsoleOutputUTF8() +end + +-- ── Aegisub shims ───────────────────────────────────────────────────────────── + +local shims = require "l0.AegisubShims" +local aegisub = shims.aegisub -- pulled into local scope; global is set by the shim for sub-modules + +-- ── Shared: workspace + DepCtrl bootstrap ──────────────────────────────────── + +local function setupDepCtrl(taskName) + local tempBase = shims.getPathToken("temp") + local workspace = tempBase .. pathSep .. ("depctrl-" .. taskName .. "-%x"):format(os.time() % 0x100000) + for _, token in ipairs({ "user", "local", "data", "temp" }) do + shims.setPathToken(token, workspace .. pathSep .. token) + end + + local FileOps = require "l0.DependencyControl.FileOps" + FileOps.mkdir("?temp", false, true) + FileOps.mkdir("?user/log", false, true) + + -- Disable the self-updater so loading DepCtrl does not trigger a network + -- fetch of its own feed (slow, flaky, pointless outside Aegisub). + local constants = require "l0.DependencyControl.Constants" + local globalConfigPath = aegisub.decode_path("?user/config/" .. constants.DEPCTRL_NAMESPACE .. ".json") + FileOps.mkdir(globalConfigPath, true, true) + do + local json = require "l0.dkjson" + local h = assert(io.open(globalConfigPath, "w")) + h:write(json.encode({ config = { updates = { mode = "off" } } })) + h:close() + end + + return require "l0.DependencyControl" +end + +-- ── Shared: feed loading, target filtering, source resolution ──────────────── + +-- Loads and expands a feed (Local mode resolves each file's on-disk source path). +local function loadFeed(feedPath) + local UpdateFeed = require "l0.DependencyControl.UpdateFeed" + local feed = UpdateFeed(nil, false, feedPath) + local ok, err = feed:loadFile(feedPath, UpdateFeed.ExpansionMode.Local) + if not ok then + io.stderr:write("Error loading feed '" .. feedPath .. "': " .. tostring(err) .. "\n") + os.exit(1) + end + return feed +end + +-- Builds a ScriptTargetFilter from the --target-module/--target-macro selectors. With no +-- selectors it includes everything; otherwise just the named packages, by type. +local function buildFilter(cliArgs) + local Common = require "l0.DependencyControl.Common" + local filter = require("l0.DependencyControl.ScriptTargetFilter")() + local mods, macros = cliArgs.target_module or {}, cliArgs.target_macro or {} + if #mods == 0 and #macros == 0 then return filter:includeAll() end + for _, ns in ipairs(mods) do filter:include(Common.ScriptType.Module, ns) end + for _, ns in ipairs(macros) do filter:include(Common.ScriptType.Automation, ns) end + return filter +end + +-- Builds a `requireId -> source path` map from every file in the feed and registers it as a +-- fallback module searcher (after the standard ones), so packages whose source layout isn't +-- namespaced (e.g. a flat repo root) still resolve straight from the checkout. Namespaced +-- repos keep resolving via the stock moonpath/path searchers, which run first. +local function registerFeedSearcher(feed) + local moonbase = require "moonscript.base" + + -- ".moon" -> "", "/Common.moon" -> ".Common", "/test/Common.moon" -> ".test.Common" + local function leafSuffix(name) + return (name:gsub("%.moon$", ""):gsub("%.lua$", ""):gsub("/", ".")) + end + + local sourceById = {} + for file, _, pkg in feed:walkFiles() do + local src = file.localFilePath + if src then + local base = file.type == "test" and (pkg.namespace .. ".test") or pkg.namespace + local id = base .. leafSuffix(file.name) + sourceById[id] = sourceById[id] or src -- first channel wins; sources are channel-agnostic + end + end + + table.insert(package.loaders or package.searchers, function(modName) + local src = sourceById[modName] + if not src then return "\n\tno source mapped in feed for '" .. modName .. "'" end + if src:match("%.moon$") then + local chunk, err = moonbase.loadfile(src) + if not chunk then error("error compiling " .. src .. ": " .. tostring(err)) end + return chunk + end + return assert(loadfile(src)) + end) + + return sourceById +end + +-- ── Command dispatch ────────────────────────────────────────────────────────── + +-- ─── test ───────────────────────────────────────────────────────────────────── +if args.command == "test" then + -- Resolve every test suite by its source require identifier, ".test". + -- Standard searchers resolve namespaced repos (e.g. DepCtrl's own modules/ tree); + -- the feed searcher registered below catches non-namespaced ones. Set before any + -- package is required, since requiring a managed module triggers test registration. + DEPCTRL_UNIT_TEST_SUITE_REQUIRE_IDENTIFIER = function(scriptType, namespace) + return namespace .. ".test" + end + + local DepCtrl = setupDepCtrl("tests") + local FileOps = require "l0.DependencyControl.FileOps" + + local feedPath = resolveAbsPath(args.feed) + local feed = loadFeed(feedPath) + + local selected = {} + for pkg, scriptType in feed:walkPackages(buildFilter(args)) do + selected[#selected + 1] = { namespace = pkg.namespace, scriptType = scriptType } + end + table.sort(selected, function(a, b) return a.namespace < b.namespace end) + if #selected == 0 then + io.stderr:write("No packages matched in feed '" .. feedPath .. "'.\n") + os.exit(1) + end + registerFeedSearcher(feed) + + local reportDir = resolveAbsPath(args.report_dir) + local ran, skipped, failed = 0, 0, 0 + local allFailures = {} -- accumulated across packages for the end-of-run summary + + for _, pkg in ipairs(selected) do + local ns = pkg.namespace + local okRequire, mod = xpcall(require, debug.traceback, ns) + local record = okRequire and DepCtrl:getRegisteredRecord(ns) or nil + + if not okRequire then + io.stderr:write(("! %s: failed to load (%s)\n"):format(ns, tostring(mod))) + failed = failed + 1 + elseif not (record and record.__class and record.__class.__name == "DependencyControl") then + io.stderr:write(("~ %s: not a DependencyControl-managed package, skipping\n"):format(ns)) + skipped = skipped + 1 + elseif record.haveTestSuite == false then + if record.testSuiteLoadError then + io.stderr:write(("! %s: test suite failed to load (%s)\n"):format(ns, tostring(record.testSuiteLoadError))) + failed = failed + 1 + else + io.stderr:write(("~ %s: no test suite found, skipping\n"):format(ns)) + end + skipped = skipped + 1 + elseif not record.testSuiteInitialized then + io.stderr:write(("! %s: test suite failed to initialize (%s)\n"):format(ns, tostring(record.testSuiteInitializeError))) + failed = failed + 1 + else + io.stdout:write(("\n=== Testing %s ===\n"):format(ns)) + local success = record.tests:run() + ran = ran + 1 + if not success then + failed = failed + 1 + for _, f in ipairs(record.tests:getFailures()) do + f.namespace = ns + allFailures[#allFailures + 1] = f + end + end + + local reportPath = FileOps.joinPath(reportDir, ns .. ".json") + local wrote, writeErr = record.tests:writeResults(reportPath) + io.stderr:write(wrote and ("Wrote CTRF report to " .. reportPath .. "\n") + or ("Warning: couldn't write CTRF report for " .. ns .. ": " .. tostring(writeErr) .. "\n")) + end + end + + if #allFailures > 0 then + io.stdout:write("\n—— Failures ——\n") + for _, f in ipairs(allFailures) do + io.stdout:write(("\n%s > %s > %s [%s]\n"):format( + f.namespace, f.classname, f.name, f.isAssertion and "assertion" or "error")) + local err = tostring(f.error or ""):gsub("%s+$", "") + io.stdout:write(" " .. err:gsub("\n", "\n ") .. "\n") + end + end + + io.stdout:write(("\n%d package(s) tested, %d skipped, %d failed.\n"):format(ran, skipped, failed)) + os.exit(failed > 0 and 1 or 0) + +-- ─── bundle ─────────────────────────────────────────────────────────────────── +elseif args.command == "bundle" then + local feedPath = resolveAbsPath(args.feed) + local outputDir = resolveAbsPath(args.out_dir) + + setupDepCtrl("bundle") + + local FileOps = require "l0.DependencyControl.FileOps" + local ZipArchiver = require "l0.DependencyControl.ZipArchiver" + local GitRepository = require "l0.DependencyControl.GitRepository" + + local feed = loadFeed(feedPath) + local filter = buildFilter(args) + + local distDir = outputDir .. pathSep .. "dist" + FileOps.remove(distDir, true) + FileOps.mkdir(distDir, false, true) + + local fileCount, errCount = feed:deployFiles(distDir, filter, false) + + -- Name the archive after the feed's headline module (DepCtrl's own feed) where present, + -- otherwise fall back to the first module version so other feeds still bundle. + local mainVersion = feed:getModuleVersion("l0.DependencyControl") + if not mainVersion then + for ns in pairs(feed.data.modules or {}) do + mainVersion = feed:getModuleVersion(ns) + if mainVersion then break end + end + mainVersion = mainVersion or "0.0.0" + end + + local suffix = GitRepository(feed.feedDir):getVersionSuffix() + local zipPath = outputDir .. pathSep .. (feed.data.name .. "-v%s%s.zip"):format(mainVersion, suffix) + + local zipOk = false + if fileCount > 0 then + local success, archiveErr = ZipArchiver(zipPath):addDirectory(distDir):write() + if success then + zipOk = true + else + io.stderr:write("Warning: archive creation failed: " .. tostring(archiveErr) .. "\n") + end + end + + local status = fileCount > 0 and "Bundle complete" or "Bundle produced no files" + io.stdout:write(("\n%s: %d file(s) in %s, %d error(s)\n"):format(status, fileCount, distDir, errCount)) + if zipOk then io.stdout:write(("Archive: %s\n"):format(zipPath)) end + os.exit(errCount > 0 and 1 or 0) + +-- ─── deploy ─────────────────────────────────────────────────────────────────── +elseif args.command == "deploy" then + local feedPath = resolveAbsPath(args.feed) + local outputDir = resolveAbsPath(args.out_dir) + local clobber = args.clobber == true + + setupDepCtrl("deploy") + + local feed = loadFeed(feedPath) + local filter = buildFilter(args) + + local fileCount, errCount = feed:deployFiles(outputDir, filter, clobber) + + local status = fileCount > 0 and "Deploy complete" or "Deploy produced no files" + io.stdout:write(("\n%s: %d file(s) deployed to %s, %d error(s)\n"):format(status, fileCount, outputDir, errCount)) + os.exit(errCount > 0 and 1 or 0) + +-- ─── update-feed ────────────────────────────────────────────────────────────── +elseif args.command == "update-feed" then + local feedPath = resolveAbsPath(args.feed) + + setupDepCtrl("update-feed") + + local UpdateFeed = require "l0.DependencyControl.UpdateFeed" + local feed = UpdateFeed(nil, false, feedPath) + + registerFeedSearcher(feed) + + local stats, err = feed:updateFeed({ + channel = args.channel, + filter = buildFilter(args), + schemaDir = table.concat({ launcherDir, "schemas", "feed" }, pathSep), + outPath = args.dry_run and false or nil, + }) + + if not stats then + io.stderr:write("Error updating feed: " .. tostring(err) .. "\n") + os.exit(1) + end + + -- Per-package breakdown: one status line per package, with any errors indented beneath it. + local changedWord = args.dry_run and "would change" or "updated" + for _, pkg in ipairs(stats.packages) do + local label = pkg.namespace .. (pkg.channel and (" (" .. pkg.channel .. ")") or "") + local status + if #pkg.errors > 0 then + status = ("%d error%s"):format(#pkg.errors, #pkg.errors == 1 and "" or "s") + elseif pkg.changed then + status = changedWord + else + status = "no changes" + end + io.stdout:write((" %-48s %s\n"):format(label, status)) + for _, e in ipairs(pkg.errors) do + io.stderr:write(" ! " .. (tostring(e):gsub("\n", "\n ")) .. "\n") + end + end + + -- Summary + local total = #stats.packages + if stats.changed > 0 then + local verb = args.dry_run and "would change — dry run, nothing written" or ("updated in " .. feedPath) + io.stdout:write(("\n%d of %d package(s) %s\n"):format(stats.changed, total, verb)) + else + io.stdout:write("\nFeed is already up to date.\n") + end + if stats.errored > 0 then + io.stdout:write(("%d package(s) had errors (see above).\n"):format(stats.errored)) + end + os.exit(stats.errored > 0 and 1 or 0) + +elseif args.command == "validate-schema" then + if args.type ~= "config" and args.type ~= "feed" then + io.stderr:write("--type must be 'config' or 'feed'.\n"); os.exit(2) + end + if not args.file then + io.stderr:write("--file is required.\n"); os.exit(2) + end + local filePath = resolveAbsPath(args.file) + + -- Loads DependencyControl, which registers its provides searcher so the vendored dkjson resolves as + -- 'json' — the module JsonSchema (and hence lua-schema) needs to parse schema files. + setupDepCtrl("validate-schema") + + local FileOps = require "l0.DependencyControl.FileOps" + local json = require "l0.dkjson" + local JsonSchema = require "l0.DependencyControl.JsonSchema" + + local raw, readErr = FileOps.readFile(filePath) + if not raw then + io.stderr:write(("Couldn't read '%s': %s\n"):format(filePath, tostring(readErr))); os.exit(1) + end + local data, _, decErr = json.decode(raw) + if type(data) ~= "table" then + io.stderr:write(("Couldn't parse '%s' as JSON: %s\n"):format(filePath, tostring(decErr or "not a JSON object"))) + os.exit(1) + end + + local schemaDir = table.concat({ launcherDir, "schemas", args.type }, pathSep) + local schemasByVersion, schemasErr = JsonSchema:getSchemasInDirectory(schemaDir) + if not schemasByVersion then + io.stderr:write(tostring(schemasErr) .. "\n"); os.exit(1) + end + + local valid, version, message + if args.schema_version then + local schemaPath = schemasByVersion[args.schema_version] + if not schemaPath then + io.stderr:write(("No %s schema for version '%s' in %s\n"):format(args.type, args.schema_version, schemaDir)) + os.exit(1) + end + version = args.schema_version + valid, message = JsonSchema(schemaPath):validate(data) + else + -- validateAny selects the schema from the file itself: a config by its root `$schema`; a feed carries no + -- `$schema`, so it's given a function that reads the feed's legacy `dependencyControlFeedFormatVersion` + -- field instead. A file with neither falls back through the available schemas, highest version first. + local hint + if args.type == "feed" then + hint = function(feed) return feed.dependencyControlFeedFormatVersion end + end + valid, version, message = JsonSchema:validateAny(data, schemasByVersion, hint) + end + + if valid then + io.stdout:write(("'%s' is valid against the %s schema v%s.\n"):format(filePath, args.type, tostring(version))) + os.exit(0) + else + io.stderr:write(("'%s' failed %s schema validation%s:\n%s\n"):format( + filePath, args.type, version and (" (v" .. version .. ")") or "", tostring(message))) + os.exit(1) + end +end diff --git a/macros/l0.DependencyControl.Toolbox.moon b/macros/l0.DependencyControl.Toolbox.moon index d66c5f6..0ba15da 100644 --- a/macros/l0.DependencyControl.Toolbox.moon +++ b/macros/l0.DependencyControl.Toolbox.moon @@ -1,34 +1,86 @@ export script_name = "DependencyControl Toolbox" export script_description = "Provides DependencyControl maintenance and configuration tools." -export script_version = "0.1.3" +export script_version = "0.3.0" export script_author = "line0" export script_namespace = "l0.DependencyControl.Toolbox" DepCtrl = require "l0.DependencyControl" -depRec = DepCtrl feed: "https://raw.githubusercontent.com/TypesettingTools/DependencyControl/master/DependencyControl.json" +Common = DepCtrl.Common +constants = require "l0.DependencyControl.Constants" +depRec = DepCtrl { + feed: "https://raw.githubusercontent.com/TypesettingTools/DependencyControl/master/DependencyControl.json", + { + {"l0.DependencyControl", version: "0.7.0"} + } +} logger = DepCtrl.logger logger.usePrefixWindow = false msgs = { install: { - scanning: "Scanning %d available feeds..." + scanning: "Scanning %d available feeds...", + createScriptUpdateRecordFailed: "Failed to create an update record for %s '%s' from feed %s: %s" } uninstall: { running: "Uninstalling %s '%s'..." - success: "%s '%s' was removed sucessfully. Reload your automation scripts or restart Aegisub for the changes to take effect." + success: "%s '%s' was removed successfully. Reload your automation scripts or restart Aegisub for the changes to take effect." lockedFiles: "%s Some script files are still in use and will be deleted during the next restart/reload:\n%s" error: "Error: %s" } + scheduleUpdatesAndRegisterTests: { + moduleLoadFailed: "Couldn't load module '%s' to schedule updates/register its tests: %s" + registerMacrosError: "Error registering test macros for module '%s': %s" + scheduleError: "Unexpected error scheduling update for record '%s': %s" + } macroConfig: { hints: { customMenu: "Lets you sort your automation macros into submenus. Use / to denote submenu levels." userFeed: "When set the updater will use this feed exclusively to update the script in question." } } + manageFeeds: { + scanning: "Fetching all feeds — this may take a moment..." + noFeeds: "DependencyControl doesn't know about any feeds yet." + openFailed: "Couldn't open a browser for %s." + sourcedWarning: "%s %s will change the update source of these installed packages:\n%s\n\nProceed?" + cantBlockBootstrap: "Refusing that block — it would match DependencyControl's own feed and disable all trust." + promptUntrusted: "This untrusted feed is advertised by another feed:\n%s\n\nFetch it to discover the feeds it lists? \"Trust\" remembers it for next time; \"Block\" hides it." + } } -- Shared Functions +FeedAction = DepCtrl.FeedManager.FeedAction + +-- Dialog button labels the code branches on, centralized so a caption can be reworded in one place without +-- touching dispatch logic — and shared with the tests through testExports, so a rename needs no test edits. +buttons = { + apply: "Apply" + close: "Close" + discover: "Fetch/Discover" + extraFeeds: "Extra Feeds" + blockList: "Block List" + help: "Help" + yes: "Yes" + no: "No" + trust: "Trust" + fetchOnce: "Fetch once" + block: "Block" + skip: "Skip" + cancel: "Cancel" +} + +-- Manage Feeds action-dropdown labels per FeedAction, with the reverse lookup that turns a picked label back +-- into its action. Shared with the tests through testExports. +feedActionLabels = { + [FeedAction.Trust]: "Trust" + [FeedAction.Block]: "Block" + [FeedAction.Unblock]: "Unblock" + [FeedAction.Remove]: "Remove" + [FeedAction.OpenBrowser]: "Browser" +} +feedActionByLabel = {label, action for action, label in pairs feedActionLabels} + buildInstalledDlgList = (scriptType, config, isUninstall) -> list, map, protectedModules = {}, {}, {} if isUninstall @@ -37,7 +89,7 @@ buildInstalledDlgList = (scriptType, config, isUninstall) -> for namespace, script in pairs config.c[scriptType] continue if protectedModules[namespace] - item = "%s v%s%s"\format script.name, depRec\getVersionString(script.version), + item = "%s v%s%s"\format script.name, DepCtrl.SemanticVersioning\toString(script.version), script.activeChannel and " [#{script.activeChannel}]" or "" list[#list+1] = item table.sort list, (a, b) -> a\lower! < b\lower! @@ -49,25 +101,18 @@ getConfig = (section) -> config.c.macros or= {} if not section or #section == 0 return config -getKnownFeeds = (config) -> - getScriptFeeds = (t) -> [v.userFeed or v.feed for _,v in pairs config.c[t] when v.feed or v.userFeed] - - -- fetch all feeds and look for further known feeds - recurse = (feeds, knownFeeds = {}, feedList = {}) -> - for url in *feeds - feed = DepCtrl.UpdateFeed url - continue if knownFeeds[url] or not feed.data - feedList[#feedList+1], knownFeeds[url] = feed, true - recurse feed\getKnownFeeds!, knownFeeds, feedList - return knownFeeds, feedList - - -- get additional feeds added by the user - knownFeeds, feedList = recurse DepCtrl.config.c.extraFeeds - -- collect feeds from all installed automation scripts and modules - recurse getScriptFeeds("modules"), knownFeeds, feedList - recurse getScriptFeeds("macros"), knownFeeds, feedList - - return feedList +-- Builds a FeedInventory over a merged config view: the installed macros/modules sections come from their +-- own handlers, while the feed lists and fetch policy are read from the live global config. Shared by the +-- install browser's feed discovery and the Manage Feeds macro. +buildFeedInventory = -> + macrosKey = Common.ScriptTypeSection[Common.ScriptType.Automation] + modulesKey = Common.ScriptTypeSection[Common.ScriptType.Module] + getSectionData = (key) -> + view = DepCtrl.config\getSectionHandler key + view and view.c or {} + mergedC = setmetatable {[macrosKey]: getSectionData(macrosKey), [modulesKey]: getSectionData(modulesKey)}, + {__index: DepCtrl.config.c} + DepCtrl.FeedInventory {c: mergedC}, DepCtrl.updater.feedTrust, DepCtrl.updater.feedLoader getScriptListDlg = (macros, modules) -> { @@ -77,12 +122,44 @@ getScriptListDlg = (macros, modules) -> {name: "module", class: "dropdown", x: 1, y: 1, width: 1, height: 1, items: modules, value: "" } } -runUpdaterTask = (scriptData, exhaustive) -> +runUpdaterTask = (scriptData, isInstall) -> return unless scriptData - task, err = DepCtrl.updater\addTask scriptData, nil, nil, exhaustive, scriptData.channel - if task then task\run! - else logger\log err + task, code, extErr = DepCtrl.updater\addTask scriptData, nil, nil, nil, scriptData.channel, + DepCtrl.Updater.UpdateReason.UserRequested + return task\run! if task + with scriptData + logger\log DepCtrl.Updater.getUpdaterErrorMsg code, .moduleName or .name, + .moduleName and Common.ScriptType.Module or Common.ScriptType.Automation, isInstall, extErr + +-- our feeds all live under raw.githubusercontent.com; abbreviate that host in the UI and expand it back on input +shortenUrl = (url) -> (url\gsub "^https://raw%.githubusercontent%.com/", "ghuc://") +expandUrl = (url) -> (url\gsub "^ghuc://", "https://raw.githubusercontent.com/") + +-- Under fetchUntrustedFeeds = "prompt", a crawl asks before fetching each untrusted feed: Trust remembers +-- the feed, Block hides it, Fetch once follows it this time only, Skip leaves it unfetched. +promptUntrustedFeed = (url, ft) -> + btn = aegisub.dialog.display { + {class: "label", x: 0, y: 0, width: 3, height: 1, label: msgs.manageFeeds.promptUntrusted\format shortenUrl url} + }, {buttons.trust, buttons.fetchOnce, buttons.block, buttons.skip}, {ok: buttons.fetchOnce, cancel: buttons.skip} + switch btn + when buttons.trust + ft\trust url + true + when buttons.fetchOnce then true + when buttons.block + ft\block url + false + else false + +-- Crawls the feed inventory with the untrusted-feed prompter active (so the `prompt` policy asks), scoped +-- so the prompter never leaks into background fetches. Shared by install discovery and Manage Feeds. +crawlWithPrompt = (inventory) -> + feedTrust = DepCtrl.updater.feedTrust + feedTrust\setPrompter promptUntrustedFeed + entries = inventory\crawl! + feedTrust\setPrompter nil + entries -- Macros @@ -90,16 +167,22 @@ install = -> config = getConfig! addAvailableToInstall = (tbl, feed, scriptType) -> - for namespace, data in pairs feed.data[scriptType] - scriptData = feed\getScript namespace, scriptType == "modules", nil, false + scriptTypeConfigAndFeedKeyName = Common.ScriptTypeSection[scriptType] + + for namespace, data in pairs feed.data[scriptTypeConfigAndFeedKeyName] + scriptData, err = feed\getScript namespace, scriptType, nil, false + if err + logger\warn msgs.install.createScriptUpdateRecordFailed\format Common.terms.scriptType.singular[scriptType], namespace, feed.url, err + continue + channels, defaultChannel = scriptData\getChannels! tbl[namespace] or= {} for channel in *channels record = scriptData.data.channels[channel] - verNum = depRec\getVersionNumber record.version - unless config.c[scriptType][namespace] or (tbl[namespace][channel] and verNum < tbl[namespace][channel].verNum) + verNum = DepCtrl.SemanticVersioning\toNumber record.version + unless config.c[scriptTypeConfigAndFeedKeyName][namespace] or (tbl[namespace][channel] and verNum < tbl[namespace][channel].verNum) tbl[namespace][channel] = { name: scriptData.name, version: record.version, verNum: verNum, feed: feed.url, - default: defaultChannel == channel, moduleName: scriptType == "modules" and namespace } + default: defaultChannel == channel, moduleName: scriptType == Common.ScriptType.Module and namespace } return tbl buildDlgList = (tbl) -> @@ -114,14 +197,19 @@ install = -> return list, map - -- get a list of the highest versions of automation scripts and modules - -- we can install but wich are not yet installed - macros, modules, feeds = {}, {}, getKnownFeeds config + -- get the highest versions of automation scripts and modules we can install but don't have yet. + -- FeedInventory crawls the known feeds (trust-gated and bounded); the shared feed loader then serves + -- each reachable feed's data from the cache the crawl just populated. + macros, modules = {}, {} + entries = crawlWithPrompt buildFeedInventory! - logger\log msgs.install.scanning, #feeds - for feed in *feeds - macros = addAvailableToInstall macros, feed, "macros" - modules = addAvailableToInstall modules, feed, "modules" + logger\log msgs.install.scanning, #entries + for entry in *entries + continue unless entry.fetched + feed = DepCtrl.updater.feedLoader\load entry.url + continue unless feed.data + macros = addAvailableToInstall macros, feed, Common.ScriptType.Automation + modules = addAvailableToInstall modules, feed, Common.ScriptType.Module -- build macro and module lists as well as reverse mappings moduleList, moduleMap = buildDlgList modules @@ -132,8 +220,8 @@ install = -> -- create and run the update tasks macro, mdl = macroMap[res.macro], moduleMap[res.module] - runUpdaterTask mdl, false - runUpdaterTask macro, false + runUpdaterTask mdl, true + runUpdaterTask macro, true uninstall = -> doUninstall = (script) -> @@ -176,22 +264,20 @@ update = -> moduleList, moduleMap = buildInstalledDlgList "modules", config macroList, macroMap = buildInstalledDlgList "macros", config - dlg = getScriptListDlg macroList, moduleList - dlg[5] = {name: "exhaustive", label: "Exhaustive Mode", class: "checkbox", x: 0, y: 2, width: 1, height: 1} - btn, res = aegisub.dialog.display dlg + btn, res = aegisub.dialog.display getScriptListDlg macroList, moduleList return unless btn -- create and run the update tasks macro, mdl = macroMap[res.macro], moduleMap[res.module] - runUpdaterTask mdl, res.exhaustive - runUpdaterTask macro, res.exhaustive + runUpdaterTask mdl, false + runUpdaterTask macro, false macroConfig = -> config = getConfig "macros" - dlg, i = {}, 1 + dlg, i = {}, 0 for nsp, macro in pairs config.userConfig - dlg[i*5+t-1] = tbl for t, tbl in ipairs { + dlg[i*5+t] = tbl for t, tbl in ipairs { {label: macro.name, class: "label", x: 0, y: i, width: 1, height: 1 }, {label: "Menu Group: ", class: "label", x: 1, y: i, width: 1, height: 1 }, {name: "#{nsp}.customMenu", class: "edit", x: 2, y: i, width: 1, height: 1, @@ -211,11 +297,314 @@ macroConfig = -> elseif v != "" config.c[nsp][prop] = v - config\write! + config\save! + +-- A simple yes/no confirmation dialog. Returns true only when the user chooses Yes. +confirmDialog = (message) -> + btn = aegisub.dialog.display {{class: "label", x: 0, y: 0, width: 1, height: 1, label: message}}, + {buttons.yes, buttons.no}, {ok: buttons.yes, cancel: buttons.no} + btn == buttons.yes + +-- Add/remove the user's extraFeeds: check feeds to drop and/or type a new one, Apply, re-open. +manageExtraFeeds = (feedTrust) -> + while true + feeds = [url for url in *(DepCtrl.config.c.feeds.extraFeeds or {})] + table.sort feeds + dlg = {} + if #feeds == 0 + dlg[#dlg + 1] = {class: "label", x: 0, y: 0, width: 3, height: 1, label: "You have no extra feeds."} + else + dlg[#dlg + 1] = {class: "label", x: 0, y: 0, width: 3, height: 1, label: "Your extra feeds:"} + for i, url in ipairs feeds + dlg[#dlg + 1] = {class: "label", x: 0, y: i, width: 2, height: 1, label: shortenUrl url} + dlg[#dlg + 1] = {class: "checkbox", x: 2, y: i, width: 1, height: 1, name: "remove#{i}", label: "Remove", value: false} + addY = #feeds + 1 + dlg[#dlg + 1] = {class: "label", x: 0, y: addY, width: 1, height: 1, label: "Add feed URL:"} + dlg[#dlg + 1] = {class: "edit", x: 1, y: addY, width: 2, height: 1, name: "newFeed", text: "", hint: "full URL or ghuc:// shorthand"} + + btn, res = aegisub.dialog.display dlg, {buttons.apply, buttons.close}, {ok: buttons.apply, cancel: buttons.close} + break if not btn or btn == buttons.close + + for i = 1, #feeds + feedTrust\removeExtraFeed feeds[i] if res["remove#{i}"] + newFeed = res.newFeed and res.newFeed\match "^%s*(.-)%s*$" + feedTrust\addExtraFeed expandUrl(newFeed) if newFeed and #newFeed > 0 + +-- Add/remove block entries. Official blocks (from DepCtrl's own feed) are read-only; user blocks can be +-- removed. New blocks match by prefix or exact URL, with an optional reason; blocking the bootstrap feed is refused. +manageBlockList = (feedTrust) -> + BlockMatchMode = DepCtrl.FeedTrust.BlockMatchMode + while true + blocks = feedTrust\getBlockedFeeds! + table.sort blocks, (a, b) -> a.url < b.url + dlg = {} + if #blocks == 0 + dlg[#dlg + 1] = {class: "label", x: 0, y: 0, width: 3, height: 1, label: "The block list is currently empty."} + else + dlg[#dlg + 1] = {class: "label", x: 0, y: 0, width: 2, height: 1, label: "Blocked feed"} + dlg[#dlg + 1] = {class: "label", x: 2, y: 0, width: 1, height: 1, label: "Mode"} + dlg[#dlg + 1] = {class: "label", x: 3, y: 0, width: 2, height: 1, label: "Reason"} + for i, entry in ipairs blocks + dlg[#dlg + 1] = {class: "label", x: 0, y: i, width: 2, height: 1, label: shortenUrl entry.url} + dlg[#dlg + 1] = {class: "label", x: 2, y: i, width: 1, height: 1, label: entry.matchMode} + dlg[#dlg + 1] = {class: "label", x: 3, y: i, width: 2, height: 1, label: entry.reason or ""} + if entry.isOfficial + dlg[#dlg + 1] = {class: "label", x: 5, y: i, width: 1, height: 1, label: "(official)"} + else + dlg[#dlg + 1] = {class: "checkbox", x: 5, y: i, width: 1, height: 1, name: "remove#{i}", label: "Remove", value: false} + addY = #blocks + 1 + dlg[#dlg + 1] = {class: "label", x: 0, y: addY, width: 1, height: 1, label: "Add block:"} + dlg[#dlg + 1] = {class: "edit", x: 1, y: addY, width: 2, height: 1, name: "newUrl", text: "", hint: "feed URL/prefix, full or ghuc://"} + dlg[#dlg + 1] = {class: "dropdown", x: 3, y: addY, width: 1, height: 1, name: "newMode", + items: {BlockMatchMode.Prefix, BlockMatchMode.Exact}, value: BlockMatchMode.Prefix} + dlg[#dlg + 1] = {class: "label", x: 0, y: addY + 1, width: 1, height: 1, label: "Reason:"} + dlg[#dlg + 1] = {class: "edit", x: 1, y: addY + 1, width: 4, height: 1, name: "newReason", text: ""} + + btn, res = aegisub.dialog.display dlg, {buttons.apply, buttons.close}, {ok: buttons.apply, cancel: buttons.close} + break if not btn or btn == buttons.close + + for i, entry in ipairs blocks + feedTrust\unblock entry.url if not entry.isOfficial and res["remove#{i}"] + newUrl = res.newUrl and res.newUrl\match "^%s*(.-)%s*$" + if newUrl and #newUrl > 0 + expanded = expandUrl newUrl + candidate = {url: expanded, matchMode: res.newMode} + if DepCtrl.FeedTrust\matchesBlockEntry constants.DEPCTRL_FEED_URL, candidate + logger\log msgs.manageFeeds.cantBlockBootstrap + else + reason = res.newReason and #res.newReason > 0 and res.newReason or nil + feedTrust\block expanded, {matchMode: res.newMode, :reason} + +-- Compact "time since" label for a Unix timestamp, for the Manage Feeds "Fetched" column. Falls back to an +-- em dash when the feed has never been fetched into the cache. +formatAge = (fetchedAt) -> + return "—" unless fetchedAt + delta = os.time! - fetchedAt + return "now" if delta < 60 + return "#{math.floor delta / 60}m" if delta < 3600 + return "#{math.floor delta / 3600}h" if delta < 86400 + return "#{math.floor delta / 86400}d" if delta < 604800 + "#{math.floor delta / 604800}w" + +-- Single source of truth for the Manage Feeds glyphs, shared by the feed-list rendering (`manageFeeds`) and +-- the Help legend (`manageFeedsHelp`). Change a glyph here and both follow. +glyphs = { + -- OK column (reachability) + reachable: "✓" + unreachable: "✗" + unknown: "?" + -- Trust column (Harvey balls: official = left half, user = right half, both = full) + untrusted: "⭘" + trustOfficial: "◐" + trustUser: "◑" + trustBoth: "⬤" + blocked: "⊘" + -- Known column + ownFeed: "★" + officialKnown: "☆" + transitive: "≫" + -- Cust column (your customizations) + extraFeed: "E" + override: "O" + -- Pkg column (a package's own feed refs) + declared: "D" + advertised: "A" + -- Use column + inUse: "↻" +} + +-- A read-only reference explaining the feed-list columns and their glyphs, opened from the Help button. +manageFeedsHelp = -> + sections = { + {"OK", "Feed reachability, filled in after fetch: ❬ #{glyphs.reachable} ❭ reachable · ❬ #{glyphs.unreachable} ❭ unreachable · ❬ #{glyphs.unknown} ❭ not checked yet"} + {"Fetched", "How long ago each feed was last fetched into the cache (e.g. 5m, 2h, 3d, 4w); ❬ — ❭ never fetched. Read from the cache, so it's shown before fetch too."} + {"Trust", "How DependencyControl treats the feed: ❬ #{glyphs.untrusted} ❭ untrusted · ❬ #{glyphs.trustOfficial} ❭ officially trusted · ❬ #{glyphs.trustUser} ❭ user-trusted · ❬ #{glyphs.trustBoth} ❭ official + user · ❬ #{glyphs.blocked} ❭ blocked"} + {"Known", "How the feed is known to DependencyControl: ❬ #{glyphs.ownFeed} ❭ its own feed · ❬ #{glyphs.officialKnown} ❭ official (listed in its feed) · ❬ #{glyphs.transitive} ❭ transitive (advertised by another feed) · ❬ #{glyphs.unknown} ❭ not checked yet"} + {"Cust", "Your customizations: ❬ #{glyphs.extraFeed} ❭ a feed in your extraFeeds · ❬ #{glyphs.override} ❭ your per-package feed override"} + {"Pkg", "A package's own feed references: ❬ #{glyphs.declared} ❭ declared by an installed package · ❬ #{glyphs.advertised} ❭ advertised by a package's dependency"} + {"Use", "Whether the feed is the effective update source of an installed package: ❬ #{glyphs.inUse} ❭ yes"} + {"Action", "Trust, Block, Unblock, Remove the feed, or open the feed in the DepCtrl Browser for details."} + } + dlg = { {class: "label", x: 0, y: 0, width: 4, height: 1, label: "Manage Feeds — what each column means"} } + for i, s in ipairs sections + dlg[#dlg + 1] = {class: "label", x: 0, y: i + 1, width: 1, height: 1, label: s[1]} + dlg[#dlg + 1] = {class: "label", x: 1, y: i + 1, width: 7, height: 1, label: s[2]} + aegisub.dialog.display dlg, {buttons.close}, {ok: buttons.close, cancel: buttons.close} + +-- Lets the user see and manage the feeds DependencyControl knows about and their trust status. The reachable +-- feeds are gathered (or crawled) via FeedInventory; FeedManager computes the per-feed actions and executes +-- them through the shared feedTrust. Aegisub's dialog toolkit has no list widget, so this uses the +-- redisplay-loop / row-grid pattern: pick an action per feed, apply, re-open. +manageFeeds = -> + feedTrust = DepCtrl.updater.feedTrust + FeedInventory = DepCtrl.FeedInventory + FeedManager = DepCtrl.FeedManager + TrustStatus = DepCtrl.FeedTrust.TrustStatus + Provenance = FeedInventory.Provenance + openUrl = require "l0.DependencyControl.helpers.open-url" + + inventory = buildFeedInventory! + manager = FeedManager feedTrust + + trustGlyphs = { + [TrustStatus.TrustedOfficial]: glyphs.trustOfficial + [TrustStatus.TrustedUser]: glyphs.trustUser + [TrustStatus.TrustedBoth]: glyphs.trustBoth + [TrustStatus.Untrusted]: glyphs.untrusted + [TrustStatus.Blocked]: glyphs.blocked + } + provenanceLabels = { + [Provenance.OfficialDepCtrl]: "DependencyControl's own feed" + [Provenance.OfficialKnown]: "known to DependencyControl" + [Provenance.UserExtra]: "in your extra feeds" + [Provenance.PackageDeclared]: "declared by an installed package" + [Provenance.PackageOverride]: "an installed package's feed override" + [Provenance.DependencyAdvertised]: "advertised by a package dependency" + [Provenance.TransitiveKnown]: "advertised by another feed" + } + describeProvenance = (row) -> table.concat [provenanceLabels[p] or p for p in *row.provenance], "; " + + -- "Known" folds official and transitive knowledge into one column: DepCtrl's own feed, an officially-known + -- feed listed in its feed, or a transitively-known feed advertised by another feed (shown only when not + -- official). Transitive knowledge isn't determined until a Discover has run, so it reads unknown until then. + getKnownGlyph = (has, row, discovered) -> + return glyphs.ownFeed if has[Provenance.OfficialDepCtrl] + return glyphs.officialKnown if has[Provenance.OfficialKnown] + return glyphs.unknown unless discovered + has[Provenance.TransitiveKnown] and glyphs.transitive or "" + + -- "Cust" (your customizations: extra feed / override) and "Pkg" (a package's own feed refs: declared / + -- advertised) groups can co-occur, so each shows letter marks for whichever are present. Every glyph is + -- called with (has, row, discovered); most use only `has`. + provColumns = { + {header: "Know", glyph: getKnownGlyph} + {header: "Cust", glyph: (has) -> (has[Provenance.UserExtra] and glyphs.extraFeed or "") .. (has[Provenance.PackageOverride] and glyphs.override or "")} + {header: "Pkg", glyph: (has) -> (has[Provenance.PackageDeclared] and glyphs.declared or "") .. (has[Provenance.DependencyAdvertised] and glyphs.advertised or "")} + {header: "Use", glyph: (has, row) -> row.inUse and glyphs.inUse or ""} + } -depRec\registerMacros{ + -- prompts for an optional block reason; returns the reason (possibly "") or nil if the user cancels + promptBlockReason = (url) -> + btn, res = aegisub.dialog.display { + {class: "label", x: 0, y: 0, width: 2, height: 1, label: "Reason for blocking #{url} (optional):"} + {class: "edit", x: 0, y: 1, width: 2, height: 1, name: "reason", text: ""} + }, {buttons.block, buttons.cancel}, {ok: buttons.block, cancel: buttons.cancel} + return nil unless btn == buttons.block + res.reason + + buildDialog = (rows, discovered) -> + feedCol = 2 -- feed URL sits after the OK (x0) and Trust (x1) columns + feedWidth = 4 + fetchedCol = feedCol + feedWidth -- when each feed was last fetched into the cache + fetchedWidth = 2 + provStart = fetchedCol + fetchedWidth + actionCol = provStart + #provColumns + dlg = { + {class: "label", x: 0, y: 0, width: 1, height: 1, label: "OK"} + {class: "label", x: 1, y: 0, width: 1, height: 1, label: "Trust"} + {class: "label", x: feedCol, y: 0, width: feedWidth, height: 1, label: "Feed"} + {class: "label", x: fetchedCol, y: 0, width: fetchedWidth, height: 1, label: "Fetched"} + {class: "label", x: actionCol, y: 0, width: 1, height: 1, label: "Action"} + } + for j, col in ipairs provColumns + dlg[#dlg + 1] = {class: "label", x: provStart + j - 1, y: 0, width: 1, height: 1, label: col.header} + for i, row in ipairs rows + has = {p, true for p in *row.provenance} + reach = discovered and (row.reachable and glyphs.reachable or glyphs.unreachable) or glyphs.unknown -- unknown until a Discover has fetched + items = {"—"} + items[#items + 1] = feedActionLabels[a] for a in *row.actions + dlg[#dlg + 1] = {class: "label", x: 0, y: i, width: 1, height: 1, label: reach} + dlg[#dlg + 1] = {class: "label", x: 1, y: i, width: 1, height: 1, label: trustGlyphs[row.trustStatus] or glyphs.unknown} + dlg[#dlg + 1] = {class: "label", x: feedCol, y: i, width: feedWidth, height: 1, label: shortenUrl row.url} + dlg[#dlg + 1] = {class: "label", x: fetchedCol, y: i, width: fetchedWidth, height: 1, label: formatAge row.lastFetchedAt} + for j, col in ipairs provColumns + dlg[#dlg + 1] = {class: "label", x: provStart + j - 1, y: i, width: 1, height: 1, label: col.glyph(has, row, discovered)} + dlg[#dlg + 1] = {class: "dropdown", x: actionCol, y: i, width: 1, height: 1, name: "action#{i}", + items: items, value: "—", hint: describeProvenance row} + dlg + + applySelections = (res, rows) -> + for i, row in ipairs rows + label = res["action#{i}"] + continue if not label or label == "—" + action = feedActionByLabel[label] + continue unless action + + if action == FeedAction.OpenBrowser + ok = openUrl.open row.browserUrl + logger\log msgs.manageFeeds.openFailed, row.url unless ok + continue + + if action == FeedAction.Block or action == FeedAction.Remove + sourced = inventory\getPackagesSourcedFrom row.url + if #sourced > 0 + verb = action == FeedAction.Block and "Blocking" or "Removing" + message = msgs.manageFeeds.sourcedWarning\format verb, row.url, table.concat(sourced, "\n") + continue unless confirmDialog message + + opts = {} + if action == FeedAction.Block + reason = promptBlockReason row.url + continue if reason == nil + opts.reason = reason if reason != "" + + manager\applyAction action, row, opts + + discovered = false + while true + logger\log msgs.manageFeeds.scanning if discovered + entries = discovered and crawlWithPrompt(inventory) or inventory\gather! + rows = FeedManager.buildRows entries + if #rows == 0 + logger\log msgs.manageFeeds.noFeeds + return + btn, res = aegisub.dialog.display buildDialog(rows, discovered), + {buttons.apply, buttons.discover, buttons.extraFeeds, buttons.blockList, buttons.help, buttons.close}, + {ok: buttons.apply, cancel: buttons.close} + break if not btn or btn == buttons.close + switch btn + when buttons.discover then discovered = true + when buttons.extraFeeds then manageExtraFeeds feedTrust + when buttons.blockList then manageBlockList feedTrust + when buttons.help then manageFeedsHelp! + else applySelections res, rows + +depRec\registerMacros { {"Install Script", "Installs an automation script or module on your system.", install}, {"Update Script", "Manually check and perform updates to any installed script.", update}, {"Uninstall Script", "Removes an automation script or module from your system.", uninstall}, + {"Manage Feeds", "See and manage the feeds DependencyControl knows about and their trust status.", manageFeeds}, {"Macro Configuration", "Lets you change per-automation script settings.", macroConfig}, -}, "DependencyControl" \ No newline at end of file +}, "DependencyControl", {:shortenUrl, :expandUrl, :formatAge, :buildInstalledDlgList, :promptUntrustedFeed, + :confirmDialog, :manageExtraFeeds, :manageBlockList, :buttons, :feedActionLabels} + +-- Force-loads all installed modules, then sweeps the live record registry to schedule +-- periodic update checks for every record and register unit test menus for modules. +-- Automation scripts schedule their own update checks on macro invocation and register +-- their own test menus within their own Aegisub environment. +scheduleUpdatesAndRegisterTests = -> + config = getConfig! + + for namespace in pairs (config.c.modules or {}) + success, err = pcall require, namespace + unless success + logger\trace msgs.scheduleUpdatesAndRegisterTests.moduleLoadFailed, namespace, tostring err + + for _, record in pairs DepCtrl\getAllRegisteredRecords! + success, errMsgOrErrCode, errDetail = pcall DepCtrl.updater\scheduleUpdate, record + if not success + logger\trace msgs.scheduleUpdatesAndRegisterTests.scheduleError, record.name or record.namespace, errMsgOrErrCode + elseif errMsgOrErrCode < 0 + logger\trace msgs.scheduleUpdatesAndRegisterTests.scheduleError, record.name or record.namespace, + DepCtrl.Updater.getUpdaterErrorMsg errMsgOrErrCode, record.name or record.namespace, record.scriptType, false, errDetail + + if record.tests and record.scriptType == Common.ScriptType.Module + success, errMsg = pcall record.tests\registerMacros + unless success + logger\trace msgs.scheduleUpdatesAndRegisterTests.registerMacrosError, record.name or record.namespace, errMsg + + DepCtrl.updater\releaseLock! + +-- The startup sweep is an Aegisub-session concern; headless (CLI/test runner) has no session to +-- schedule for, and running it would trigger live update checks while a test require is in flight. +scheduleUpdatesAndRegisterTests! unless Common.isHeadless! diff --git a/macros/l0.DependencyControl.Toolbox/test.moon b/macros/l0.DependencyControl.Toolbox/test.moon new file mode 100644 index 0000000..1c94e9d --- /dev/null +++ b/macros/l0.DependencyControl.Toolbox/test.moon @@ -0,0 +1,419 @@ +UnitTestSuite = require "l0.DependencyControl.UnitTestSuite" +constants = require "l0.DependencyControl.Constants" +DepCtrl = require "l0.DependencyControl" + +-- Suite for the DependencyControl Toolbox macro. The Plumbing class proves the automation-script test +-- wiring (DependencyControl discovers the suite and hands it the script's registered macros + testExports); +-- the Urls/Age/InstalledList/UntrustedPrompt classes cover the pure and near-pure helpers; the remaining +-- classes drive the dialog flows end to end, stubbing aegisub.dialog.display and the Toolbox's collaborators +-- (the updater, config handler, feed inventory/manager, and the injected feed trust model). +UnitTestSuite "l0.DependencyControl.Toolbox", (macros, dependencies, testExports, controls) -> + {:shortenUrl, :expandUrl, :formatAge, :buildInstalledDlgList, :promptUntrustedFeed, + :confirmDialog, :manageExtraFeeds, :manageBlockList, :buttons, :feedActionLabels} = testExports + + -- a fake config the way buildInstalledDlgList reads it: config.c[section] is the installed-package map + makeConfig = (section, entries) -> {c: {[section]: entries}} + -- the set of script display names present in a buildInstalledDlgList map + namesIn = (map) -> {v.name, true for _, v in pairs map} + + -- Stub aegisub.dialog.display to return a queued sequence of results, one per call: each entry is a + -- {button, values} pair (or {false} to model a cancel). Exhausting the queue returns nil, which the + -- redisplay loops read as "close". Returns the stub for call assertions. + queueDialog = (ut, responses) -> + i = 0 + ut\stub(aegisub.dialog, "display")\calls (...) -> + i += 1 + r = responses[i] or {} + unpack r, 1, #r + + -- a fake FeedTrust recording the mutations the trust panels delegate to it; getBlockedFeeds hands back a + -- fresh copy each call so a panel's in-place sort can't corrupt the source list across redisplay cycles + makeFeedTrust = (blocked) -> + calls = {block: {}, unblock: {}, trust: {}, addExtraFeed: {}, removeExtraFeed: {}} + { + :calls + getBlockedFeeds: => [b for b in *(blocked or {})] + block: (url, opts) => calls.block[#calls.block + 1] = {url, opts} + unblock: (url) => calls.unblock[#calls.unblock + 1] = url + trust: (url) => calls.trust[#calls.trust + 1] = url + addExtraFeed: (url) => calls.addExtraFeed[#calls.addExtraFeed + 1] = url + removeExtraFeed: (url) => calls.removeExtraFeed[#calls.removeExtraFeed + 1] = url + } + + -- a minimal fake UpdateFeed exposing one installable automation script, enough for install discovery + makeFakeFeed = -> + scriptData = { + name: "X" + data: {channels: {main: {version: "1.0.0"}}} + getChannels: => {"main"}, "main" + } + { + url: "feed://x" + data: {macros: {"a.x": true}, modules: {}} + getScript: (ns, st) => scriptData, nil + } + + { + Plumbing: { + _description: "Automation-script test wiring: registered macros and testExports reach the suite." + + -- every macro registered through registerMacros is exposed by name, carrying its unhooked process + receivesRegisteredMacros: (ut) -> + for name in *{"Install Script", "Update Script", "Uninstall Script", "Manage Feeds", "Macro Configuration"} + ut\assertNotNil macros[name] + ut\assertFunction macros[name].process + + -- the script's own internal helpers, passed straight through as testExports + receivesTestExports: (ut) -> + ut\assertNotNil testExports + ut\assertFunction testExports.shortenUrl + ut\assertFunction testExports.expandUrl + + _order: {"receivesRegisteredMacros", "receivesTestExports"} + } + + Urls: { + _description: "shortenUrl/expandUrl: raw.githubusercontent.com <-> ghuc:// abbreviation." + + -- the raw.githubusercontent.com host collapses to the ghuc:// scheme + shorten_abbreviatesHost: (ut) -> + ut\assertEquals shortenUrl("https://raw.githubusercontent.com/TT/DepCtrl/master/x.json"), + "ghuc://TT/DepCtrl/master/x.json" + + -- a URL on any other host is left untouched + shorten_passesThroughOther: (ut) -> + ut\assertEquals shortenUrl("https://example.com/feed.json"), "https://example.com/feed.json" + + -- ghuc:// expands back to the full raw.githubusercontent.com URL + expand_restoresHost: (ut) -> + ut\assertEquals expandUrl("ghuc://TT/DepCtrl/master/x.json"), + "https://raw.githubusercontent.com/TT/DepCtrl/master/x.json" + + -- shorten then expand round-trips a raw.githubusercontent.com URL unchanged + roundTrips: (ut) -> + url = "https://raw.githubusercontent.com/TT/DepCtrl/master/x.json" + ut\assertEquals expandUrl(shortenUrl url), url + + _order: {"shorten_abbreviatesHost", "shorten_passesThroughOther", "expand_restoresHost", "roundTrips"} + } + + Age: { + _description: "formatAge: compact 'time since' label for a feed's last-fetch timestamp." + + -- an absent timestamp (never fetched) renders as an em dash + never_isDash: (ut) -> ut\assertEquals formatAge(nil), "—" + + -- under a minute reads as 'now'; larger spans round down into the coarsest unit that fits + picksCoarsestUnit: (ut) -> + now = os.time! + ut\assertEquals formatAge(now), "now" + ut\assertEquals formatAge(now - 120), "2m" + ut\assertEquals formatAge(now - 7200), "2h" + ut\assertEquals formatAge(now - 172800), "2d" + ut\assertEquals formatAge(now - 1209600), "2w" + + _order: {"never_isDash", "picksCoarsestUnit"} + } + + InstalledList: { + _description: "buildInstalledDlgList: installed-package picker rows, with the uninstall protected-module guard." + + -- uninstall must not offer DependencyControl's own stack (its namespace is protected) + uninstall_excludesProtected: (ut) -> + config = makeConfig "modules", { + [constants.DEPCTRL_NAMESPACE]: {name: "DepCtrl", version: "0.7.0"} + "l0.Other": {name: "Other", version: "1.2.3"} + } + list, map = buildInstalledDlgList "modules", config, true + names = namesIn map + ut\assertNil names["DepCtrl"] -- protected on uninstall + ut\assertTrue names["Other"] + ut\assertEquals #list, 1 + + -- install/update (not an uninstall) protects nothing, so every installed package is offered + install_includesAll: (ut) -> + config = makeConfig "modules", { + [constants.DEPCTRL_NAMESPACE]: {name: "DepCtrl", version: "0.7.0"} + "l0.Other": {name: "Other", version: "1.2.3"} + } + list, map = buildInstalledDlgList "modules", config, false + names = namesIn map + ut\assertTrue names["DepCtrl"] + ut\assertTrue names["Other"] + ut\assertEquals #list, 2 + + -- rows are sorted case-insensitively by display name + sortsByNameCaseInsensitively: (ut) -> + config = makeConfig "macros", { + "a.z": {name: "Zebra", version: "1.0.0"} + "a.a": {name: "apple", version: "1.0.0"} + "a.m": {name: "Mango", version: "1.0.0"} + } + list = buildInstalledDlgList "macros", config, false + ut\assertEquals list, {"apple v1.0.0", "Mango v1.0.0", "Zebra v1.0.0"} + + -- a package with a non-default active channel shows the channel in brackets + formatsActiveChannel: (ut) -> + config = makeConfig "macros", {"a.x": {name: "X", version: "2.0.0", activeChannel: "beta"}} + list = buildInstalledDlgList "macros", config, false + ut\assertEquals list[1], "X v2.0.0 [beta]" + + _order: {"uninstall_excludesProtected", "install_includesAll", "sortsByNameCaseInsensitively", "formatsActiveChannel"} + } + + UntrustedPrompt: { + _description: "promptUntrustedFeed: maps the untrusted-feed dialog choice to a fetch decision + trust side effect." + + -- Trust: remembers the feed and fetches it + trust_remembersAndFetches: (ut) -> + ft = makeFeedTrust! + ut\stub(aegisub.dialog, "display")\calls -> buttons.trust + ut\assertTrue promptUntrustedFeed "feed://x", ft + ut\assertEquals ft.calls.trust, {"feed://x"} + ut\assertEquals #ft.calls.block, 0 + + -- Fetch once: fetches without remembering + fetchOnce_fetchesWithoutTrusting: (ut) -> + ft = makeFeedTrust! + ut\stub(aegisub.dialog, "display")\calls -> buttons.fetchOnce + ut\assertTrue promptUntrustedFeed "feed://x", ft + ut\assertEquals #ft.calls.trust, 0 + ut\assertEquals #ft.calls.block, 0 + + -- Block: blocks the feed and refuses the fetch + block_blocksAndRefuses: (ut) -> + ft = makeFeedTrust! + ut\stub(aegisub.dialog, "display")\calls -> buttons.block + ut\assertFalse promptUntrustedFeed "feed://x", ft + ut\assertEquals ft.calls.block[1][1], "feed://x" + ut\assertEquals #ft.calls.trust, 0 + + -- Skip (and dialog cancel): refuses the fetch, changing no trust state + skip_refusesQuietly: (ut) -> + ft = makeFeedTrust! + ut\stub(aegisub.dialog, "display")\calls -> buttons.skip + ut\assertFalse promptUntrustedFeed "feed://x", ft + ut\assertEquals #ft.calls.trust, 0 + ut\assertEquals #ft.calls.block, 0 + + _order: {"trust_remembersAndFetches", "fetchOnce_fetchesWithoutTrusting", "block_blocksAndRefuses", "skip_refusesQuietly"} + } + + Confirm: { + _description: "confirmDialog: a yes/no gate that is true only on Yes." + + yes_isTrue: (ut) -> + ut\stub(aegisub.dialog, "display")\calls -> buttons.yes + ut\assertTrue confirmDialog "proceed?" + + no_isFalse: (ut) -> + ut\stub(aegisub.dialog, "display")\calls -> buttons.no + ut\assertFalse confirmDialog "proceed?" + + _order: {"yes_isTrue", "no_isFalse"} + } + + BlockList: { + _description: "manageBlockList: the block-list panel — remove user blocks, add a block, refuse blocking the bootstrap feed." + + -- Close on the first display makes no changes + close_makesNoChanges: (ut) -> + ft = makeFeedTrust {{url: "feed://x", isOfficial: false, matchMode: DepCtrl.FeedTrust.BlockMatchMode.Prefix}} + queueDialog ut, {{buttons.close}} + manageBlockList ft + ut\assertEquals #ft.calls.unblock, 0 + ut\assertEquals #ft.calls.block, 0 + + -- a checked user block is unblocked; a checked official block is left alone (it has no remove control) + removesUserBlockNotOfficial: (ut) -> + ft = makeFeedTrust { + {url: "feed://official", isOfficial: true, matchMode: DepCtrl.FeedTrust.BlockMatchMode.Prefix} + {url: "feed://user", isOfficial: false, matchMode: DepCtrl.FeedTrust.BlockMatchMode.Prefix} + } + -- sorted by URL: [feed://official, feed://user] -> remove1 targets the official (ignored), remove2 the user + queueDialog ut, {{buttons.apply, {remove1: true, remove2: true}}, {buttons.close}} + manageBlockList ft + ut\assertEquals ft.calls.unblock, {"feed://user"} + + -- a typed URL is added as a block carrying the chosen mode and reason + addsBlockWithReason: (ut) -> + ft = makeFeedTrust {} + queueDialog ut, { + {buttons.apply, {newUrl: "ghuc://bad/feed", newMode: DepCtrl.FeedTrust.BlockMatchMode.Exact, newReason: "malware"}} + {buttons.close} + } + manageBlockList ft + ut\assertEquals ft.calls.block[1][1], "https://raw.githubusercontent.com/bad/feed" + ut\assertEquals ft.calls.block[1][2].matchMode, DepCtrl.FeedTrust.BlockMatchMode.Exact + ut\assertEquals ft.calls.block[1][2].reason, "malware" + + -- blocking DependencyControl's own bootstrap feed is refused (it would collapse the trust root) + refusesBootstrapBlock: (ut) -> + ft = makeFeedTrust {} + queueDialog ut, { + {buttons.apply, {newUrl: constants.DEPCTRL_FEED_URL, newMode: DepCtrl.FeedTrust.BlockMatchMode.Exact}} + {buttons.close} + } + manageBlockList ft + ut\assertEquals #ft.calls.block, 0 + + _order: {"close_makesNoChanges", "removesUserBlockNotOfficial", "addsBlockWithReason", "refusesBootstrapBlock"} + } + + ExtraFeeds: { + _description: "manageExtraFeeds: the discovery-roots panel — add a typed feed (expanding ghuc://), close cleanly." + + close_makesNoChanges: (ut) -> + ft = makeFeedTrust! + queueDialog ut, {{buttons.close}} + manageExtraFeeds ft + ut\assertEquals #ft.calls.addExtraFeed, 0 + ut\assertEquals #ft.calls.removeExtraFeed, 0 + + -- a typed ghuc:// shorthand is trimmed and expanded before being added as a discovery root + addsTypedFeedExpanded: (ut) -> + ft = makeFeedTrust! + queueDialog ut, {{buttons.apply, {newFeed: " ghuc://x/y "}}, {buttons.close}} + manageExtraFeeds ft + ut\assertEquals ft.calls.addExtraFeed, {"https://raw.githubusercontent.com/x/y"} + ut\assertEquals #ft.calls.removeExtraFeed, 0 + + _order: {"close_makesNoChanges", "addsTypedFeedExpanded"} + } + + Update: { + _description: "Update Script: picks installed packages and runs update tasks for the selection." + + -- cancelling the picker runs no update task + cancel_runsNoTask: (ut) -> + ran = {} + ut\stub(DepCtrl.config, "getSectionHandler")\returns {c: {macros: {"a.x": {name: "X", version: "1.0.0"}}, modules: {}}} + ut\stub(DepCtrl.updater, "addTask")\calls (self, sd) -> ran[#ran + 1] = sd + queueDialog ut, {{false}} + macros["Update Script"].process! + ut\assertEquals #ran, 0 + + -- selecting a macro runs an update task for exactly that package + selectsMacro_runsItsTask: (ut) -> + macroEntry = {name: "X", version: "1.0.0"} + ran = {} + ut\stub(DepCtrl.config, "getSectionHandler")\returns {c: {macros: {"a.x": macroEntry}, modules: {}}} + ut\stub(DepCtrl.updater, "addTask")\calls (self, sd) -> + ran[#ran + 1] = sd + {run: => true} + queueDialog ut, {{"OK", {macro: "X v1.0.0", module: ""}}} + macros["Update Script"].process! + ut\assertEquals #ran, 1 + ut\assertEquals ran[1], macroEntry + + _order: {"cancel_runsNoTask", "selectsMacro_runsItsTask"} + } + + Uninstall: { + _description: "Uninstall Script: builds the (protected-guarded) installed list and cancels cleanly." + + -- cancelling the picker uninstalls nothing (and never constructs a record to uninstall) + cancel_uninstallsNothing: (ut) -> + ut\stub(DepCtrl.config, "getSectionHandler")\returns {c: {macros: {"a.x": {name: "X", version: "1.0.0"}}, modules: {}}} + dlg = queueDialog ut, {{false}} + macros["Uninstall Script"].process! + dlg\assertCalledOnce! + + _order: {"cancel_uninstallsNothing"} + } + + Install: { + _description: "Install Script: crawls feeds for installable packages and runs install tasks for the selection." + + -- an unreachable (unfetched) feed contributes nothing; cancelling installs nothing + skipsUnfetchedAndCancels: (ut) -> + ran = {} + ut\stub(DepCtrl.config, "getSectionHandler")\returns {c: {macros: {}, modules: {}}} + crawl = ut\stub(DepCtrl.FeedInventory.__base, "crawl")\calls -> {{url: "feed://x", fetched: false}} + ut\stub(DepCtrl.updater, "addTask")\calls (self, sd) -> ran[#ran + 1] = sd + queueDialog ut, {{false}} + macros["Install Script"].process! + crawl\assertCalled! + ut\assertEquals #ran, 0 + + -- a fetched feed's available script is offered, and selecting it runs an install task for it + selectsAvailableScript: (ut) -> + ran = {} + ut\stub(DepCtrl.config, "getSectionHandler")\returns {c: {macros: {}, modules: {}}} + ut\stub(DepCtrl.FeedInventory.__base, "crawl")\calls -> {{url: "feed://x", fetched: true}} + ut\stub(DepCtrl.FeedLoader.__base, "load")\calls -> makeFakeFeed! + ut\stub(DepCtrl.updater, "addTask")\calls (self, sd) -> + ran[#ran + 1] = sd + {run: => true} + queueDialog ut, {{"OK", {macro: "X v1.0.0", module: ""}}} + macros["Install Script"].process! + ut\assertEquals #ran, 1 + ut\assertEquals ran[1].namespace, "a.x" + ut\assertEquals ran[1].feed, "feed://x" + + _order: {"skipsUnfetchedAndCancels", "selectsAvailableScript"} + } + + MacroConfig: { + _description: "Macro Configuration: writes non-empty edits into the per-macro config and saves." + + appliesEditsAndSaves: (ut) -> + saved = {} + cfg = { + userConfig: {"a.x": {name: "X"}} + c: {"a.x": {}} + save: => saved.yes = true + } + ut\stub(DepCtrl.config, "getSectionHandler")\returns cfg + dlg = queueDialog ut, {{"OK", {"a.x.customMenu": "Tools/Mine", "a.x.userFeed": ""}}} + macros["Macro Configuration"].process! + ut\assertEquals cfg.c["a.x"].customMenu, "Tools/Mine" -- non-empty edit written + ut\assertNil cfg.c["a.x"].userFeed -- empty edit left unset + ut\assertTrue saved.yes + -- the built dialog is a gap-free array whose first row sits at y=0 (no wasted leading row) + built = dlg._calls[1][1] + ut\assertNotNil built[1] + ut\assertEquals built[1].y, 0 + + _order: {"appliesEditsAndSaves"} + } + + ManageFeeds: { + _description: "Manage Feeds: the redisplay loop dispatches Apply to feed actions and Discover to the crawl." + + -- Close on the first render applies no feed action + close_appliesNothing: (ut) -> + rows = {{url: "feed://x", provenance: {}, trustStatus: DepCtrl.FeedTrust.TrustStatus.Untrusted, actions: {}, browserUrl: "http://b"}} + ut\stub(DepCtrl.FeedManager, "buildRows")\calls -> rows + applyAction = ut\stub DepCtrl.FeedManager.__base, "applyAction" + queueDialog ut, {{buttons.close}} + macros["Manage Feeds"].process! + applyAction\assertNotCalled! + + -- Apply dispatches each row's chosen action to the feed manager + apply_dispatchesSelectedAction: (ut) -> + Trust = DepCtrl.FeedManager.FeedAction.Trust + row = {url: "feed://x", provenance: {}, trustStatus: DepCtrl.FeedTrust.TrustStatus.Untrusted, actions: {Trust}, browserUrl: "http://b"} + applied = {} + ut\stub(DepCtrl.FeedManager, "buildRows")\calls -> {row} + ut\stub(DepCtrl.FeedManager.__base, "applyAction")\calls (self, action, entry) -> + applied[#applied + 1] = {action, entry} + true + queueDialog ut, {{buttons.apply, {action1: feedActionLabels[Trust]}}, {buttons.close}} + macros["Manage Feeds"].process! + ut\assertEquals #applied, 1 + ut\assertEquals applied[1][1], Trust + ut\assertEquals applied[1][2], row + + -- Fetch/Discover switches the list source from the offline gather to the (here stubbed) network crawl + discover_usesCrawl: (ut) -> + rows = {{url: "feed://x", provenance: {}, trustStatus: DepCtrl.FeedTrust.TrustStatus.Untrusted, actions: {}, browserUrl: "http://b"}} + ut\stub(DepCtrl.FeedManager, "buildRows")\calls -> rows + crawl = ut\stub(DepCtrl.FeedInventory.__base, "crawl")\calls -> {} + queueDialog ut, {{buttons.discover}, {buttons.close}} + macros["Manage Feeds"].process! + crawl\assertCalled! + + _order: {"close_appliesNothing", "apply_dispatchesSelectedAction", "discover_usesCrawl"} + } + } diff --git a/modules/DependencyControl.moon b/modules/DependencyControl.moon deleted file mode 100644 index b7a95c4..0000000 --- a/modules/DependencyControl.moon +++ /dev/null @@ -1,49 +0,0 @@ -MIN_MOONSCRIPT_VERSION = "0.3.0" - -SemanticVersioning = require "l0.DependencyControl.SemanticVersioning" -moonscript = require 'moonscript.version' -assert SemanticVersioning\check(moonscript.version, MIN_MOONSCRIPT_VERSION), - [[ DependencyControl requires Moonscript v%s or later to work, -however the Version %s provided by your Aegisub installation is outdated. -Update to a recent Aegisub build to resolve this issue. -]]\format MIN_MOONSCRIPT_VERSION, moonscript.version - - -Logger = require "l0.DependencyControl.Logger" -UpdateFeed = require "l0.DependencyControl.UpdateFeed" -ConfigHandler = require "l0.DependencyControl.ConfigHandler" -FileOps = require "l0.DependencyControl.FileOps" -Updater = require "l0.DependencyControl.Updater" -UnitTestSuite = require "l0.DependencyControl.UnitTestSuite" -Record = require "l0.DependencyControl.Record" - -class DependencyControl extends Record - @ConfigHandler = ConfigHandler - @UpdateFeed = UpdateFeed - @Logger = Logger - @Updater = Updater - @UnitTestSuite = UnitTestSuite - @FileOps = FileOps - - -rec = DependencyControl{ - name: "DependencyControl", - version: "0.6.3", - description: "Provides script management and auto-updating for Aegisub macros and modules.", - author: "line0", - url: "http://github.com/TypesettingTools/DependencyControl", - moduleName: "l0.DependencyControl", - feed: "https://raw.githubusercontent.com/TypesettingTools/DependencyControl/master/DependencyControl.json", - { - {"DM.DownloadManager", version: "0.3.1", feed: "https://raw.githubusercontent.com/torque/ffi-experiments/master/DependencyControl.json"}, - {"BM.BadMutex", version: "0.1.3", feed: "https://raw.githubusercontent.com/torque/ffi-experiments/master/DependencyControl.json"}, - {"PT.PreciseTimer", version: "0.1.5", feed: "https://raw.githubusercontent.com/torque/ffi-experiments/master/DependencyControl.json"}, - {"requireffi.requireffi", version: "0.1.1", feed: "https://raw.githubusercontent.com/torque/ffi-experiments/master/DependencyControl.json"}, - } -} -DependencyControl.__class.version = rec -LOADED_MODULES[rec.moduleName], package.loaded[rec.moduleName] = DependencyControl, DependencyControl -DependencyControl.updater\scheduleUpdate rec -rec\requireModules! - -return DependencyControl \ No newline at end of file diff --git a/modules/DependencyControl/Common.moon b/modules/DependencyControl/Common.moon deleted file mode 100644 index 8de8024..0000000 --- a/modules/DependencyControl/Common.moon +++ /dev/null @@ -1,42 +0,0 @@ -ffi = require "ffi" - -class DependencyControlCommon - -- Some terms are shared across components - @platform = "#{ffi.os}-#{ffi.arch}" - - @terms = { - scriptType: { - singular: { "automation script", "module" } - plural: { "automation scripts", "modules" } - } - - isInstall: { - [true]: "installation" - [false]: "update" - } - - capitalize: (str) -> str[1]\upper! .. str\sub 2 - } - - -- Common enums - @RecordType = { - Managed: 1 - Unmanaged: 2 - } - - @ScriptType = { - Automation: 1 - Module: 2 - name: { - legacy: { "macros", "modules" } - canonical: {"automation", "modules"} - } - } - - automationDir: { - aegisub.decode_path("?user/automation/autoload"), - aegisub.decode_path("?user/automation/include") - } - - @testDir = {aegisub.decode_path("?user/automation/tests/DepUnit/macros"), - aegisub.decode_path("?user/automation/tests/DepUnit/modules")} \ No newline at end of file diff --git a/modules/DependencyControl/ConfigHandler.moon b/modules/DependencyControl/ConfigHandler.moon deleted file mode 100644 index 11cfacf..0000000 --- a/modules/DependencyControl/ConfigHandler.moon +++ /dev/null @@ -1,330 +0,0 @@ -util = require "aegisub.util" -json = require "json" -PreciseTimer = require "PT.PreciseTimer" -mutex = require "BM.BadMutex" - -fileOps = require "l0.DependencyControl.FileOps" -Logger = require "l0.DependencyControl.Logger" - -class ConfigHandler - @handlers = {} - errors = { - jsonDecode: "JSON parse error: %s" - configCorrupted: [[An error occured while parsing the JSON config file. -A backup of the corrupted configuration has been written to '%s'. -Reload your automation scripts to generate a new configuration file.]] - badKey: "Can't %s section because the key #%d (%s) leads to a %s." - jsonRoot: "JSON root element must be an array or a hashtable, got a %s." - noFile: "No config file defined." - failedLock: "Failed to lock config file for %s: %s" - waitLockFailed: "Error waiting for existing lock to be released: %s" - forceReleaseFailed: "Failed to force-release existing lock after timeout had passed (%s)" - noLock: "#{@@__name} doesn't have a lock" - writeFailedRead: "Failed reading config file: %s." - lockTimeout: "Timeout reached while waiting for write lock." - } - traceMsgs = { - -- waitingLockPre: "Waiting %d ms before trying to get a lock..." - waitingLock: "Waiting for config file lock to be released (%d ms passed)... " - waitingLockFinished: "Lock was released after %d ms." - mergeSectionStart: "Merging own section into configuration. Own Section: %s\nConfiguration: %s" - mergeSectionResult: "Merge completed with result: %s" - fileNotFound: "Couldn't find config file '%s'." - fileCreate: "Config file '%s' doesn't exist, yet. Will write a fresh copy containing the current configuration section." - writing: "Writing config file '%s'..." - -- waitingLockTimeout: "Timeout was reached after %d seconds, force-releasing lock..." - } - - new: (@file, defaults, @section, noLoad, @logger = Logger fileBaseName: @@__name) => - @section = {@section} if "table" != type @section - @defaults = defaults and util.deep_copy(defaults) or {} - -- register all handlers for concerted writing - @setFile @file - - -- set up user configuration and make defaults accessible - @userConfig = {} - @config = setmetatable {}, { - __index: (_, k) -> - if @userConfig and @userConfig[k] ~= nil - return @userConfig[k] - else return @defaults[k] - __newindex: (_, k, v) -> - @userConfig or= {} - @userConfig[k] = v - __len: (tbl) -> return 0 - __ipairs: (tbl) -> error "numerically indexed config hive keys are not supported" - __pairs: (tbl) -> - merged = util.copy @defaults - merged[k] = v for k, v in pairs @userConfig - return next, merged - } - @c = @config -- shortcut - - -- rig defaults in a way that writing to contained tables deep-copies the whole default - -- into the user configuration and sets the requested property there - recurse = (tbl) -> - for k,v in pairs tbl - continue if type(v)~="table" or type(k)=="string" and k\match "^__" - -- replace every table reference with an empty proxy table - -- this ensures all writes to the table get intercepted - tbl[k] = setmetatable {__key: k, __parent: tbl, __tbl: v}, { - -- make the original table the index of the proxy so that defaults can be read - __index: v - __len: (tbl) -> return #tbl.__tbl - __newindex: (tbl, k, v) -> - upKeys, parent = {}, tbl.__parent - -- trace back to defaults entry, pick up the keys along the path - while parent.__parent - tbl = parent - upKeys[#upKeys+1] = tbl.__key - parent = tbl.__parent - - -- deep copy the whole defaults node into the user configuration - -- (util.deep_copy does not copy attached metatable references) - -- make sure we copy the actual table, not the proxy - @userConfig or= {} - @userConfig[tbl.__key] = util.deep_copy @defaults[tbl.__key].__tbl - -- finally perform requested write on userdata - tbl = @userConfig[tbl.__key] - for i = #upKeys-1, 1, -1 - tbl = tbl[upKeys[i]] - tbl[k] = v - __pairs: (tbl) -> return next, tbl.__tbl - __ipairs: (tbl) -> - i, n, orgTbl = 0, #tbl.__tbl, tbl.__tbl - -> - i += 1 - return i, orgTbl[i] if i <= n - } - recurse tbl[k] - - recurse @defaults - @load! unless noLoad - - setFile: (path) => - return false unless path - if @@handlers[path] - table.insert @@handlers[path], @ - else @@handlers[path] = {@} - path, err = fileOps.validateFullPath path, true - return nil, err unless path - @file = path - return true - - unsetFile: => - handlers = @@handlers[@file] - if handlers and #handlers>1 - @@handlers[@file] = [handler for handler in *handlers when handler != @] - else @@handlers[@file] = nil - @file = nil - return true - - readFile: (file = @file, useLock = true, waitLockTime) => - if useLock - time, err = @getLock waitLockTime - unless time - -- handle\close! - return false, errors.failedLock\format "reading", err - - mode, file = fileOps.attributes file, "mode" - if mode == nil - @releaseLock! if useLock - return false, file - elseif not mode - @releaseLock! if useLock - @logger\trace traceMsgs.fileNotFound, @file - return nil - - handle, err = io.open file, "r" - unless handle - @releaseLock! if useLock - return false, err - - data = handle\read "*a" - success, result = pcall json.decode, data - unless success - handle\close! - -- JSON parse error usually points to a corrupted config file - -- Rename the broken file to allow generating a new one - -- so the user can continue his work - @logger\trace errors.jsonDecode, result - backup = @file .. ".corrupted" - fileOps.copy @file, backup - fileOps.remove @file, false, true - - @releaseLock! if useLock - return false, errors.configCorrupted\format backup - - handle\close! - @releaseLock! if useLock - - if "table" != type result - return false, errors.jsonRoot\format type result - - return result - - load: => - return false, errors.noFile unless @file - - config, err = @readFile! - return config, err unless config - - sectionExists = true - for i=1, #@section - config = config[@section[i]] - switch type config - when "table" continue - when "nil" - config, sectionExists = {}, false - break - else return false, errors.badKey\format "retrive", i, tostring(@section[i]),type config - - @userConfig or= {} - @userConfig[k] = v for k,v in pairs config - return sectionExists - - mergeSection: (config) => - --@logger\trace traceMsgs.mergeSectionStart, @logger\dumpToString(@section), - -- @logger\dumpToString config - - section, sectionExists = config, true - -- create missing parent sections - for i=1, #@section - childSection = section[@section[i]] - if childSection == nil - -- don't create parent sections if this section is going to be deleted - unless @userConfig - sectionExists = false - break - section[@section[i]] = {} - childSection = section[@section[i]] - elseif "table" != type childSection - return false, errors.badKey\format "update", i, tostring(@section[i]),type childSection - section = childSection if @userConfig or i < #@section - -- merge our values into our section - if @userConfig - section[k] = v for k,v in pairs @userConfig - elseif sectionExists - section[@section[#@section]] = nil - - -- @logger\trace traceMsgs.mergeSectionResult, @logger\dumpToString config - return config - - delete: (concertWrite, waitLockTime) => - @userConfig = nil - return @write concertWrite, waitLockTime - - write: (concertWrite, waitLockTime) => - return false, errors.noFile unless @file - - -- get a lock to avoid concurrent config file access - time, err = @getLock waitLockTime - unless time - return false, errors.failedLock\format "writing", err - - -- read the config file - config, err = @readFile @file, false - if config == false - @releaseLock! - return false, errors.writeFailedRead\format err - @logger\trace traceMsgs.fileCreate, @file unless config - config or= {} - - -- merge in our section - -- concerted writing allows us to update a configuration file - -- shared by multiple handlers in the lua environment - handlers = concertWrite and @@handlers[@file] or {@} - for handler in *handlers - config, err = handler\mergeSection config - unless config - @releaseLock! - return false, err - - -- create JSON - success, res = pcall json.encode, config - unless success - @releaseLock! - return false, res - - -- write the whole config file in one go - handle, err = io.open(@file, "w") - unless handle - @releaseLock! - return false, err - - @logger\trace traceMsgs.writing, @file - handle\setvbuf "full", 10e6 - handle\write res - handle\flush! - handle\close! - @releaseLock! - - return true - - getLock: (waitTimeout = 5000, checkInterval = 50) => - return 0 if @hasLock - success = mutex.tryLock! - if success - @hasLock = true - return 0 - - timeout, timePassed = waitTimeout, 0 - while not success and timeout > 0 - PreciseTimer.sleep checkInterval - success = mutex.tryLock! - timeout -= checkInterval - timePassed = waitTimeout - timeout - if timePassed % (checkInterval*5) == 0 - @logger\trace traceMsgs.waitingLock, timePassed - - if success - @logger\trace traceMsgs.waitingLockFinished, timePassed - @hasLock = true - return timePassed - else - -- @logger\trace traceMsgs.waitingLockTimeout, waitTimeout/1000 - -- success, err = @releaseLock true - -- unless success - -- return false, errors.forceReleaseFailed\format err - -- @hasLock = true - --return waitTimeout - return false, errors.lockTimeout - - getSectionHandler: (section, defaults, noLoad) => - return @@ @file, defaults, section, noLoad, @logger - - releaseLock: (force) => - if @hasLock or force - @hasLock = false - mutex.unlock! - return true - return false, errors.noLock - - -- copied from Aegisub util.moon, adjusted to skip private keys - deepCopy: (tbl) => - seen = {} - copy = (val) -> - return val if type(val) != 'table' - return seen[val] if seen[val] - seen[val] = val - {k, copy(v) for k, v in pairs val when type(k) != "string" or k\sub(1,1) != "_"} - copy tbl - - import: (tbl = {}, keys, updateOnly, skipSameLengthTables) => - tbl = tbl.userConfig if tbl.__class == @@ - changesMade = false - @userConfig or= {} - keys = {key, true for key in *keys} if keys - - for k,v in pairs tbl - continue if keys and not keys[k] or @userConfig[k] == v - continue if updateOnly and @c[k] == nil - -- TODO: deep-compare tables - isTable = type(v) == "table" - if isTable and skipSameLengthTables and type(@userConfig[k]) == "table" and #v == #@userConfig[k] - continue - continue if type(k) == "string" and k\sub(1,1) == "_" - @userConfig[k] = isTable and @deepCopy(v) or v - changesMade = true - - return changesMade \ No newline at end of file diff --git a/modules/DependencyControl/FileOps.moon b/modules/DependencyControl/FileOps.moon deleted file mode 100644 index 1369a60..0000000 --- a/modules/DependencyControl/FileOps.moon +++ /dev/null @@ -1,307 +0,0 @@ -ffi = require "ffi" -re = require "aegisub.re" -lfs = require "lfs" - -Logger = require "l0.DependencyControl.Logger" -local ConfigHandler - -class FileOps - msgs = { - generic: { - deletionRescheduled: "Another deletion attempt has been rescheduled for the next restart." - } - attributes: { - badPath: "Path failed verification: %s." - genericError: "Can't retrieve attributes: %s." - noAttribute: "Can't find attriubte with name '%s'." - } - - mkdir: { - createError: "Error creating directory: %s." - otherExists: "Couldn't create directory because a %s of the same name is already present." - } - copy: { - targetExists: "Target file '%s' already exists" - genericError: "An error occured while copying file '%s' to '%s':\n%s" - dirCopyUnsupported: "Copying directories is currently not supported." - missingSource: "Couldn't find source file '%s'." - openError: "Couldn't open %s file '%s' for reading: \n%s" - } - move: { - inUseTryingRename: "Target file '%s' already exists and appears to be in use. Trying to rename and delete existing file..." - renamedDeletionFailed: "The existing file was successfully renamed to '%s', but couldn't be deleted (%s).\n%s" - overwritingFile: "File '%s' already exists, overwriting..." - createdDir: "Created target directory '%s'." - exists: "Couldn't move file '%s' to '%s' because a %s of the same name is already present." - genericError: "An error occured while moving file '%s' to '%s':\n%s" - createDirError: "Moving '%s' to '%s' failed (%s)." - cantRemove: "Couldn't overwrite file '%s': %s. Attempts at renaming the existing target file failed." - cantRenameTryingCopy: "Move operation failed to rename '%s' to '%s' (%s), trying copy+remove instead..." - couldntRemoveFiles: "Move operation suceeded to copied the file(s) to the target location, but some of the source files couldn't be removed:\n%s\n%s" - cantCopy: "Move operation failed to copy '%s' to '%s' (%s) after a failed rename attempt (%s)." - } - rmdir: { - emptyPath: "Argument #1 (path) must not be an empty string." - couldntRemoveFiles: "Some of the files and folders in the specified directory couldn't be removed:\n%s" - couldntRemoveDir: "Error removing empty directory: %s." - - } - validateFullPath: { - badType: "Argument #1 (path) had the wrong type. Expected 'string', got '%s'." - tooLong: "The specified path exceeded the maximum length limit (%d > %d)." - invalidChars: "The specifed path contains one or more invalid characters: '%s'." - reservedNames: "The specified path contains reserved path or file names: '%s'." - parentPath: "Accessing parent directories is not allowed." - notFullPath: "The specified path is not a valid full path." - missingExt: "The specified path is missing a file extension." - } - } - - devPattern = ffi.os == "Windows" and "[A-Za-z]:" or "/[^\\\\/]+" - pathMatch = { - sep: ffi.os == "Windows" and "\\" or "/" - pattern: re.compile "^(#{devPattern})((?:[\\\\/][^\\\\/]*[^\\\\/\\s\\.])*)[\\\\/]([^\\\\/]*[^\\\\/\\s\\.])?$" - invalidChars: '[<>:"|%?%*%z%c;]' - reservedNames: re.compile "[\\\\/](CON|COM[1-9]|PRN|AUX|NUL|LPT[1-9])(?:[\\\\/].*?)?$", re.ICASE - maxLen: 255 - } - @logger = Logger! - - createConfig = (noLoad, configDir) -> - FileOps.configDir = configDir if configDir - ConfigHandler or= require "l0.DependencyControl.ConfigHandler" - FileOps.config or= ConfigHandler "#{FileOps.configDir}/l0.#{FileOps.__name}.json", - {toRemove: {}}, nil, noLoad, FileOps.logger - return FileOps.config - - remove: (paths, recurse, reSchedule) -> - config = createConfig true - configLoaded, overallSuccess, details, firstErr = false, true, {} - paths = {paths} unless type(paths) == "table" - - for path in *paths - mode, path = FileOps.attributes path, "mode" - if mode - rmFunc = mode == "file" and os.remove or FileOps.rmdir - res, err = rmFunc path, recurse - unless res - firstErr or= err - unless reSchedule -- delete operation failed entirely - details[path] = {nil, err} - overallSuccess = nil - continue - - -- load the FileOps configuration file and reschedule deletions - unless configLoaded - FileOps.config\load! - configLoaded = true - config.c.toRemove[path] = os.time! - -- mark the operations as failed "for now", indicating a second attempt has been scheduled - details[path] = {false, err} - overallSuccess = false - - -- delete operation succeeded - else details[path] = {true} - -- file not found or permission issue - else details[path] = {nil, path} - - config\write! if configLoaded - return overallSuccess, details, firstErr - - runScheduledRemoval: (configDir) -> - config = createConfig false, configDir - paths = [path for path, _ in pairs config.c.toRemove] - if #paths > 0 - -- rescheduled removals will not be rescheduled another time - FileOps.remove paths, true - config.c.toRemove = {} - config\write! - return true - - copy: ( source, target ) -> - -- source check - mode, sourceFullPath, _, _, fileName = FileOps.attributes source, "mode" - switch mode - when "directory" - return false, msgs.copy.dirCopyUnsupported - when nil - return false, msgs.copy.genericError\format source, target, sourceFullPath - when false - return false, msgs.copy.missingSource\format source - - -- target check - checkTarget = (target) -> - mode, targetFullPath = FileOps.attributes target, "mode" - switch mode - when "file" - return false, msgs.copy.targetExists\format target - when nil - return false, msgs.copy.genericError\format source, target, targetFullPath - when "directory" - target ..= "/#{fileName}" - return checkTarget target - return true, targetFullPath - - success, targetFullPath = checkTarget target - return false, targetFullPath unless success - - input, msg = io.open sourceFullPath, "rb" - unless input - return false, msgs.copy.openError\format "source", sourceFullPath, msg - - output, msg = io.open targetFullPath, "wb" - unless output - input\close! - return false, msgs.copy.openError\format "target", targetFullPath, msg - - success, msg = output\write input\read "*a" - input\close! - output\close! - - if success - return true - else - return false, msgs.copy.genericError\format sourceFullPath, targetFullPath, msg - - - move: (source, target, overwrite) -> - mode, err = FileOps.attributes target, "mode" - if mode == "file" - unless overwrite - return false, msgs.move.exists\format source, target, mode - FileOps.logger\trace msgs.move.overwritingFile, target - res, _, err = FileOps.remove target - unless res - -- can't remove old target file, probably in use or lack of permissions - -- try to rename and then delete it - FileOps.logger\debug msgs.move.inUseTryingRename, target - junkName = "#{target}.depCtrlRemoved" - -- There might be an old removed file we couldn't delete before - FileOps.remove junkName - res = os.rename target, junkName - unless res - return false, msgs.move.cantRemove\format target, err - -- rename succeeded, now clean up after ourselves - res, _, err = FileOps.remove junkName, false, true - unless res - FileOps.logger\debug msgs.move.renamedDeletionFailed, junkName, err, msgs.generic.deletionRescheduled - - elseif mode -- a directory (or something else) of the same name as the target file is already present - return false, msgs.move.exists\format source, target, mode - elseif mode == nil -- if retrieving the attributes of a file fails, something is probably wrong - return false, msgs.move.genericError\format source, target, err - - else -- target file not found, check directory - res, dir = FileOps.mkdir target, true - if res == nil - return false, msgs.move.createDirError\format source, target, err - elseif res - FileOps.logger\trace msgs.move.createdDir, dir - - -- at this point the target directory exists and the target file doesn't, move the file - res, err = os.rename source, target - unless res - -- renaming the file failed, could be because of a permission issue - -- but me might a well be trying to rename over file system boundaries on *nix - -- so we should try copy + remove before giving up - FileOps.logger\debug msgs.move.cantRenameTryingCopy, source, target, err - renErr, res, err = err, FileOps.copy source, target - unless res - return false, msgs.move.cantCopy\format source, target, err, renErr - res, details = FileOps.remove source, false, true -- TODO: also support directories/recursion, but also require copy to support it - - unless res - fileList = table.concat ["#{path}: #{res[2]}" for path, res in pairs details when not res[1]], "\n" - FileOps.logger\debug msgs.move.couldntRemoveFiles, fileList, msgs.generic.deletionRescheduled - - return true - - rmdir: (path, recurse = true) -> - return nil, msgs.rmdir.emptyPath if path == "" - mode, path = FileOps.attributes path, "mode" - return nil, msgs.rmdir.notPath unless mode == "directory" - - if recurse - -- recursively remove contained files and directories - toRemove = ["#{path}/#{file}" for file in lfs.dir path] - res, details = FileOps.remove toRemove, true - unless res - fileList = table.concat ["#{path}: #{res[2]}" for path, res in pairs details when not res[1]], "\n" - return nil, msgs.rmdir.couldntRemoveFiles\format fileList - - -- remove empty directory - success, err = lfs.rmdir path - unless success - return nil, msgs.rmdir.couldntRemoveDir\format err - - return true - - mkdir: (path, isFile) -> - mode, fullPath, dev, dir, file = FileOps.attributes path, "mode" - dir = isFile and table.concat({dev,dir or file}) or fullPath - - if mode == nil - return nil, msgs.attributes.genericError\format fullPath - elseif not mode - res, err = lfs.mkdir dir - if err -- can't create directory (possibly a permission error) - return nil, msgs.mkdir.createError\format err - return true, dir - elseif isFile and mode == "file" -- if the file already exists, so does the directory - return false, dir - elseif mode != "directory" -- a file of the same name as the target directory is already present - return nil, msgs.mkdir.otherExists\format mode - return false, dir - - attributes: (path, key) -> - fullPath, dev, dir, file = FileOps.validateFullPath path - unless fullPath - path = "#{lfs.currentdir!}/#{path}" - fullPath, dev, dir, file = FileOps.validateFullPath path - unless fullPath - return nil, msgs.attributes.badPath\format dev - - attr, err = lfs.attributes fullPath, key - if err - return nil, msgs.attributes.genericError\format err - elseif not attr - return false, fullPath, dev, dir, file - - return attr, fullPath, dev, dir, file - - validateFullPath: (path, checkFileExt) -> - if type(path) != "string" - return nil, msgs.validateFullPath.badType\format type(path) - -- expand aegisub path specifiers - path = aegisub.decode_path path - -- expand home directory on linux - homeDir = os.getenv "HOME" - path = path\gsub "^~", "{#homeDir}/" if homeDir - -- use single native path separators - path = path\gsub "[\\/]+", pathMatch.sep - -- check length - if #path > pathMatch.maxLen - return false, msgs.validateFullPath.tooLong\format #path, pathMatch.maxLen - -- check for invalid characters - invChar = path\match pathMatch.invalidChars, ffi.os == "Windows" and 3 or nil - if invChar - return false, msgs.validateFullPath.invalidChars\format invChar - -- check for reserved file names - reserved = pathMatch.reservedNames\match path - if reserved - return false, msgs.validateFullPath.reservedNames\format reserved[2].str - -- check for path escalation - if path\match "%.%." - return false, msgs.validateFullPath.parentPath - - -- check if we got a valid full path - matches = pathMatch.pattern\match path - dev, dir, file = matches[2].str, matches[3].str, matches[4].str if matches - unless dev - return false, msgs.validateFullPath.notFullPath - if checkFileExt and not (file and file\match ".+%.+") - return false, msgs.validateFullPath.missingExt - - path = table.concat({dev, dir, file and pathMatch.sep, file}) - - return path, dev, dir, file \ No newline at end of file diff --git a/modules/DependencyControl/Record.moon b/modules/DependencyControl/Record.moon deleted file mode 100644 index 5293321..0000000 --- a/modules/DependencyControl/Record.moon +++ /dev/null @@ -1,315 +0,0 @@ -json = require "json" -lfs = require "lfs" -re = require "aegisub.re" - -Common = require "l0.DependencyControl.Common" -Logger = require "l0.DependencyControl.Logger" -ConfigHandler = require "l0.DependencyControl.ConfigHandler" -FileOps = require "l0.DependencyControl.FileOps" -Updater = require "l0.DependencyControl.Updater" -ModuleLoader = require "l0.DependencyControl.ModuleLoader" -SemanticVersioning = require "l0.DependencyControl.SemanticVersioning" - -class Record extends Common - namespaceValidation = re.compile "^(?:[-\\w]+\\.)+[-\\w]+$" - - msgs = { - new: { - badRecordError: "Error: Bad #{@@__name} record (%s)." - badRecord: { - noUnmanagedMacros: "Creating unmanaged version records for macros is not allowed" - missingNamespace: "No namespace defined" - badVersion: "Couldn't parse version number: %s" - badNamespace: "Namespace '%s' failed validation. Namespace rules: must contain 1+ single dots, but not start or end with a dot; all other characters must be in [A-Za-z0-9-_]." - badModuleTable: "Invalid required module table #%d (%s)." - } - } - uninstall: { - noVirtualOrUnmanaged: "Can't uninstall %s %s '%s'. (Only installed scripts managed by #{@@__name} can be uninstalled)." - } - writeConfig: { - error: "An error occured while writing the #{@@__name} config file: %s" - writing: "Writing updated %s data to config file..." - } - } - - @depConf = { - file: aegisub.decode_path "?user/config/l0.#{@@__name}.json", - scriptFields: {"author", "configFile", "feed", "moduleName", "name", "namespace", "url", -- REMOVE - "requiredModules", "version", "unmanaged"}, - globalDefaults: {updaterEnabled:true, updateInterval:302400, traceLevel:3, extraFeeds:{}, - tryAllFeeds:false, dumpFeeds:true, configDir:"?user/config", - logMaxFiles: 200, logMaxAge: 604800, logMaxSize:10*(10^6), - updateWaitTimeout: 60, updateOrphanTimeout: 600, - logDir: "?user/log", writeLogs: true} - } - - init = => - FileOps.mkdir @depConf.file, true - @loadConfig! - @logger = Logger { fileBaseName: "DepCtrl", fileSubName: script_namespace, prefix: "[#{@@__name}] ", - toFile: @config.c.writeLogs, defaultLevel: @config.c.traceLevel, - maxAge: @config.c.logMaxAge,maxSize: @config.c.logMaxSize, maxFiles: @config.c.logMaxFiles, - logDir: @config.c.logDir } - - @updater = Updater script_namespace, @config, @logger - @configDir = @config.c.configDir - - FileOps.mkdir aegisub.decode_path @configDir - logsHaveBeenTrimmed or= @logger\trimFiles! - FileOps.runScheduledRemoval @configDir - - - new: (args) => - init Record unless @@logger - - -- defaults - args[k] = v for k, v in pairs { - readGlobalScriptVars: true - saveRecordToConfig: true - } when args[k] == nil - - {@requiredModules, moduleName:@moduleName, configFile:configFile, virtual:@virtual, :name, - description:@description, url:@url, feed:@feed, recordType:@recordType, :namespace, - author:@author, :version, configFile:@configFile, - :readGlobalScriptVars, :saveRecordToConfig} = args - - @recordType or= @@RecordType.Managed - -- also support name key (as used in configuration) for required modules - @requiredModules or= args.requiredModules - - if @moduleName - @namespace = @moduleName - @name = name or @moduleName - @scriptType = @@ScriptType.Module - ModuleLoader.createDummyRef @ unless @virtual or @recordType == @@RecordType.Unmanaged - - else - if @virtual or not readGlobalScriptVars - @name = name or namespace - @namespace = namespace - version or= 0 - else - @name = name or script_name - @description or= script_description - @author or= script_author - version or= script_version - - @namespace = namespace or script_namespace - assert @recordType == @@RecordType.Managed, msgs.new.badRecordError\format msgs.new.badRecord.noUnmanagedMacros - assert @namespace, msgs.new.badRecordError\format msgs.new.badRecord.missingNamespace - @scriptType = @@ScriptType.Automation - - -- if the hosting macro doesn't have a namespace defined, define it for - -- the first DepCtrled module loaded by the macro or its required modules - unless script_namespace - export script_namespace = @namespace - - -- non-depctrl record don't need to conform to namespace rules - assert @virtual or @recordType == @@RecordType.Unmanaged or @validateNamespace!, - msgs.new.badRecord.badNamespace\format @namespace - - @configFile = configFile or "#{@namespace}.json" - @automationDir = @@automationDir[@scriptType] - @testDir = @@testDir[@scriptType] - @version, err = @@parseVersion version - assert @version, msgs.new.badRecordError\format msgs.new.badRecord.badVersion\format err - - @requiredModules or= {} - -- normalize short format module tables - for i, mdl in pairs @requiredModules - switch type mdl - when "table" - mdl.moduleName or= mdl[1] - mdl[1] = nil - when "string" - @requiredModules[i] = {moduleName: mdl} - else error msgs.new.badRecordError\format msgs.new.badRecord.badModuleTable\format i, tostring mdl - - shouldWriteConfig = @loadConfig! - - -- write config file if contents are missing or are out of sync with the script version record - -- ramp up the random wait time on first initialization (many scripts may want to write configuration data) - -- we can't really profit from write concerting here because we don't know which module loads last - @writeConfig if shouldWriteConfig and saveRecordToConfig - - checkOptionalModules: ModuleLoader.checkOptionalModules - - -- loads the DependencyControl global configuration - @loadConfig = => - if @config - @config\load! - else @config = ConfigHandler @depConf.file, @depConf.globalDefaults, {"config"}, nil, @logger - - -- loads the script configuration - loadConfig: (importRecord = false) => - -- virtual modules are not yet present on the user's system and have no persistent configuration - @config or= ConfigHandler not @virtual and @@depConf.file, {}, - { @@ScriptType.name.legacy[@scriptType], @namespace }, true, @@logger - - -- import and overwrites version record from the configuration - if importRecord - -- check if a module that was previously virtual was installed in the meantime - -- TODO: prevent issues caused by orphaned config entries - haveConfig = false - if @virtual - @config\setFile @@depConf.file - if @config\load! - haveConfig, @virtual = true, false - else @config\unsetFile! - else - haveConfig = @config\load! - - -- only need to refresh data if the record was changed by an update - if haveConfig - @[key] = @config.c[key] for key in *@@depConf.scriptFields - - elseif not @virtual - -- copy script information to the config - @config\load! - shouldWriteConfig = @config\import @, @@depConf.scriptFields, false, true - return shouldWriteConfig - - return false - - writeConfig: => - unless @virtual or @config.file - @config\setFile @@depConf.file - - @@logger\trace msgs.writeConfig.writing, @@terms.scriptType.singular[@scriptType] - @config\import @, @@depConf.scriptFields, false, true - success, errMsg = @config\write false - - assert success, msgs.writeConfig.error\format errMsg - - - @parseVersion = SemanticVersioning.parse - - - @getVersionString = SemanticVersioning.toString - - - getConfigFileName: () => - return aegisub.decode_path "#{@@configDir}/#{@configFile}" - - getConfigHandler: (defaults, section, noLoad) => - return ConfigHandler @getConfigFileName!, defaults, section, noLoad - - getLogger: (args = {}) => - args.fileBaseName or= @namespace - args.toFile = @config.c.logToFile if args.toFile == nil - args.defaultLevel or= @config.c.logLevel - args.prefix or= @moduleName and "[#{@name}]" - - return Logger args - - checkVersion: (value, precision = "patch") => - if type(value) == "table" and value.__class == @@ - value = value.version - return SemanticVersioning\check @version, value - - - getSubmodules: => - return nil if @virtual or @recordType == @@RecordType.Unmanaged or @scriptType != @@ScriptType.Module - mdlConfig = @@config\getSectionHandler @@ScriptType.name.legacy[@@ScriptType.Module] - pattern = "^#{@namespace}."\gsub "%.", "%%." - return [mdl for mdl, _ in pairs mdlConfig.c when mdl\match pattern], mdlConfig - - requireModules: (modules = @requiredModules, addFeeds = {@feed}) => - success, err = ModuleLoader.loadModules @, modules, addFeeds - @@updater\releaseLock! - unless success - -- if we failed loading our required modules - -- then that means we also failed to load - LOADED_MODULES[@namespace] = nil - @@logger\error err - return unpack [mdl._ref for mdl in *modules] - - registerTests: (...) => - -- load external tests - haveTests, tests = pcall require, "DepUnit.#{@@ScriptType.name.legacy[@scriptType]}.#{@namespace}" - - if haveTests and not @testsLoaded - @tests, tests.name = tests, @name - modules = table.pack @requireModules! - if @moduleName - @tests\import @ref, modules, ... - else @tests\import modules, ... - - @tests\registerMacros! - @testsLoaded = true - - register: (selfRef, ...) => - -- replace dummy refs with real refs to own module - @ref.__index, @ref, LOADED_MODULES[@moduleName] = selfRef, selfRef, selfRef - @registerTests selfRef, ... - return selfRef - - registerMacro: (name=@name, description=@description, process, validate, isActive, submenu) => - -- alternative signature takes name and description from script - if type(name)=="function" - process, validate, isActive, submenu = name, description, process, validate - name, description = @name, @description - - -- use automation script name for submenu by default - submenu = @name if submenu == true - - menuName = { @config.c.customMenu } - menuName[#menuName+1] = submenu if submenu - menuName[#menuName+1] = name - - -- check for updates before running a macro - processHooked = (sub, sel, act) -> - @@updater\scheduleUpdate @ - @@updater\releaseLock! - return process sub, sel, act - - aegisub.register_macro table.concat(menuName, "/"), description, processHooked, validate, isActive - - registerMacros: (macros = {}, submenuDefault = true) => - for macro in *macros - -- allow macro table to omit name and description - submenuIdx = type(macro[1])=="function" and 4 or 6 - macro[submenuIdx] = submenuDefault if macro[submenuIdx] == nil - @registerMacro unpack(macro, 1, 6) - - setVersion: (version) => - version, err = @@parseVersion version - if version - @version = version - return version - else return nil, err - - validateNamespace: (namespace = @namespace, isVirtual = @virtual) => - return isVirtual or namespaceValidation\match @namespace - - uninstall: (removeConfig = true) => - if @virtual or @recordType == @@RecordType.Unmanaged - return nil, msgs.uninstall.noVirtualOrUnmanaged\format @virtual and "virtual" or "unmanaged", - @@terms.scriptType.singular[@scriptType], - @name - @config\delete! - subModules, mdlConfig = @getSubmodules! - -- uninstalling a module also removes all submodules - if subModules and #subModules > 0 - mdlConfig.c[mdl] = nil for mdl in *subModules - mdlConfig\write! - - toRemove, pattern, dir = {} - if @moduleName - nsp, name = @namespace\match "(.+)%.(.+)" - pattern = "^#{name}" - dir = "#{@automationDir}/#{nsp\gsub '%.', '/'}" - else - pattern = "^#{@namespace}"\gsub "%.", "%%." - dir = @automationDir - - lfs.chdir dir - for file in lfs.dir dir - mode, path = FileOps.attributes file, "mode" - -- parent level module files must be .ext - currPattern = @moduleName and mode == "file" and pattern.."%." or pattern - -- automation scripts don't use any subdirectories - if (@moduleName or mode == "file") and file\match currPattern - toRemove[#toRemove+1] = path - return FileOps.remove toRemove, true, true \ No newline at end of file diff --git a/modules/DependencyControl/UnitTestSuite.moon b/modules/DependencyControl/UnitTestSuite.moon deleted file mode 100644 index 64b9301..0000000 --- a/modules/DependencyControl/UnitTestSuite.moon +++ /dev/null @@ -1,843 +0,0 @@ - -Logger = require "l0.DependencyControl.Logger" -re = require "aegisub.re" --- make sure tests can be loaded from the test directory -package.path ..= aegisub.decode_path("?user/automation/tests") .. "/?.lua;" - ---- A class for all single unit tests. --- Provides useful assertion and logging methods for a user-specified test function. --- @classmod UnitTest -class UnitTest - @msgs = { - run: { - setup: "Performing setup... " - teardown: "Performing teardown... " - test: "Running test '%s'... " - ok: "OK." - failed: "FAILED!" - reason: "Reason: %s" - } - new: { - badTestName: "Test name must be of type %s, got a %s." - } - - assert: { - true: "Expected true, actual value was %s." - false: "Expected false, actual value was %s." - nil: "Expected nil, actual value was %s." - notNil: "Got nil when a value was expected." - truthy: "Expected a truthy value, actual value was falsy (%s)." - falsy: "Expected a falsy value, actual value was truthy (%s)." - type: "Expected a value of type %s, actual value was of type %s." - sameType: "Type of expected value (%s) didn't match type of actual value (%s)." - inRange: "Expected value to be in range [%d .. %d], actual value %d was %s %d." - almostEquals: "Expected value to be almost equal %d ± %d, actual value was %d." - notAlmostEquals: "Expected numerical value to not be close to %d ± %d, actual value was %d." - checkArgTypes: "Expected argument #%d (%s) to be of type %s, got a %s." - zero: "Expected 0, actual value was a %s." - notZero: "Got a 0 when a number other than 0 was expected." - compare: "Expected value to be a number %s %d, actual value was %d." - integer: "Expected numerical value to be an integer, actual value was %d." - positiveNegative: "Expected a %s number (0 %s), actual value was %d." - equals: "Actual value didn't match expected value.\n%s actual: %s\n%s expected: %s" - notEquals: "Actual value equals expected value when it wasn't supposed to:\n%s actual: %s" - is: "Expected %s, actual value was %s." - isNot: "Actual value %s was identical to the expected value when it wasn't supposed to." - itemsEqual: "Actual item values of table weren't %s to the expected values (checked %s):\n Actual: %s\nExpected: %s" - itemsEqualNumericKeys: "only continuous numerical keys" - itemsEqualAllKeys: "all keys" - continuous: "Expected table to have continuous numerical keys, but value at index %d of %d was a nil." - matches: "String value '%s' didn't match expected %s pattern '%s'." - contains: "String value '%s' didn't contain expected substring '%s' (case-%s comparison)." - error: "Expected function to throw an error but it succesfully returned %d values: %s" - errorMsgMatches: "Error message '%s' didn't match expected %s pattern '%s'." - } - - formatTemplate: { - type: "'%s' of type %s" - } - } - - --- Creates a single unit test. - -- Instead of calling this constructor you'd usually provide test data - -- in a table structure to @{UnitTestSuite:new} as an argument. - -- @tparam string name a descriptive title for the test - -- @tparam function(UnitTest, ...) testFunc the function containing the test code - -- @tparam UnitTestClass testClass the test class this test belongs to - -- @treturn UnitTest the unit test - -- @see UnitTestSuite:new - new: (@name, @f = -> , @testClass) => - @logger = @testClass.logger - error type(@logger) unless type(@logger) == "table" - @logger\assert type(@name) == "string", @@msgs.new.badTestName, type @name - - --- Runs the unit test function. - -- In addition to the @{UnitTest} object itself, it also passes - -- the specified arguments into the function. - -- @param[opt] args any optional modules or other data the test function needs - -- @treturn[1] boolean true (test succeeded) - -- @treturn[2] boolean false (test failed) - -- @treturn[2] string the error message describing how the test failed - run: (...) => - @assertFailed = false - @logStart! - @success, res = xpcall @f, debug.traceback, @, ... - @logResult res - - return @success, @errMsg - - --- Formats and writes a "running test x" message to the log. - -- @local - logStart: => - @logger\logEx nil, @@msgs.run.test, false, nil, nil, @name - - --- Formats and writes the test result to the log. - -- In case of failure the message contains details about either the test assertion that failed - -- or a stack trace if the test ran into a different exception. - -- @local - -- @tparam[opt=errMsg] the error message being logged; defaults to the error returned by the last run of this test - logResult: (errMsg = @errMsg) => - if @success - @logger\logEx nil, @@msgs.run.ok, nil, nil, 0 - else - if @assertFailed - -- scrub useless stack trace from asserts provided by this module - errMsg = errMsg\gsub "%[%w+ \".-\"%]:%d+:", "" - errMsg = errMsg\gsub "stack traceback:.*", "" - @errMsg = errMsg - @logger\logEx nil, @@msgs.run.failed, nil, nil, 0 - @logger.indent += 1 - @logger\log @@msgs.run.reason, @errMsg - @logger.indent -= 1 - - --- Formats a message with a specified predefined template. - -- Currently only supports the "type" template. - -- @local - -- @tparam string template the name of the template to use - -- @param[opt] args any arguments required for formatting the message - format: (tmpl, ...) => - inArgs = table.pack ... - outArgs = switch tmpl - when "type" then {tostring(inArgs[1]), type(inArgs[1])} - - @@msgs.formatTemplate[tmpl]\format unpack outArgs - - - -- static helper functions - - --- Compares equality of two specified arguments - -- Requirements for values are considered equal: - -- [1] their types match - -- [2] their metatables are equal - -- [3] strings and numbers are compared by value - -- functions and cdata are compared by reference - -- tables must have equal values at identical indexes and are compared recursively - -- (i.e. two table copies of `{"a", {"b"}}` are considered equal) - -- @static - -- @param a the first value - -- @param b the second value - -- @tparam[opt] string aType if already known, specify the type of the first value - -- for a small performance benefit - -- @tparam[opt] string bType the type of the second value - -- @treturn boolean `true` if a and b are equal, otherwise `false` - equals: (a, b, aType, bType) -> - -- TODO: support equality comparison of tables used as keys - treeA, treeB, depth = {}, {}, 0 - - recurse = (a, b, aType = type a, bType) -> - -- identical values are equal - return true if a == b - -- only tables can be equal without also being identical - bType or= type b - return false if aType != bType or aType != "table" - - -- perform table equality comparison - return false if #a != #b - - aFieldCnt, bFieldCnt = 0, 0 - local tablesSeenAtKeys - - depth += 1 - treeA[depth], treeB[depth] = a, b - - for k, v in pairs a - vType = type v - if vType == "table" - -- comparing tables is expensive so we should keep a list - -- of keys we can skip checking when iterating table b - tablesSeenAtKeys or= {} - tablesSeenAtKeys[k] = true - - -- detect synchronous circular references to prevent infinite recursion loops - for i = 1, depth - return true if v == treeA[i] and b[k] == treeB[i] - - unless recurse v, b[k], vType - depth -= 1 - return false - - aFieldCnt += 1 - - for k, v in pairs b - continue if tablesSeenAtKeys and tablesSeenAtKeys[k] - if bFieldCnt == aFieldCnt or not recurse v, a[k] - -- no need to check further if the field count is not identical - depth -= 1 - return false - bFieldCnt += 1 - - -- check metatables for equality - res = recurse getmetatable(a), getmetatable b - depth -= 1 - return res - - return recurse a, b, aType, bType - - - --- Compares equality of two specified tables ignoring table keys. - -- The table comparison works much in the same way as @{UnitTest:equals}, - -- however this method doesn't require table keys to be equal between a and b - -- and considers two tables to be equal if an equal value is found in b for every value in a and vice versa. - -- By default this only looks at numerical indexes - -- as this kind of comparison doesn't usually make much sense for hashtables. - -- @static - -- @tparam table a the first table - -- @tparam table b the second table - -- @tparam[opt=true] bool onlyNumericalKeys Disable this option to also compare items with non-numerical keys - -- at the expense of a performance hit. - -- @tparam[opt=false] bool ignoreExtraAItems Enable this option to make the comparison one-sided, - -- ignoring additional items present in a but not in b. - -- @tparam[opt=false] bool requireIdenticalItems Enable this option if you require table items to be identical, - -- i.e. compared by reference, rather than by equality. - itemsEqual: (a, b, onlyNumKeys = true, ignoreExtraAItems, requireIdenticalItems) -> - seen, aTbls = {}, {} - aCnt, aTblCnt, bCnt = 0, 0, 0 - - findEqualTable = (bTbl) -> - for i, aTbl in ipairs aTbls - if UnitTest.equals aTbl, bTbl - table.remove aTbls, i - seen[aTbl] = nil - return true - return false - - if onlyNumKeys - aCnt, bCnt = #a, #b - return false if not ignoreExtraAItems and aCnt != bCnt - - for v in *a - seen[v] = true - if "table" == type v - aTblCnt += 1 - aTbls[aTblCnt] = v - - for v in *b - -- identical values - if seen[v] - seen[v] = nil - continue - - -- equal values - if type(v) != "table" or requireIdenticalItems or not findEqualTable v - return false - - - else - for _, v in pairs a - aCnt += 1 - seen[v] = true - if "table" == type v - aTblCnt += 1 - aTbls[aTblCnt] = v - - for _, v in pairs b - bCnt += 1 - -- identical values - if seen[v] - seen[v] = nil - continue - - -- equal values - if type(v) != "table" or requireIdenticalItems or not findEqualTable v - return false - - return false if not ignoreExtraAItems and aCnt != bCnt - - return true - - --- Helper method to mark a test as failed by assertion and throw a specified error message. - -- @local - -- @param condition passing in a falsy value causes the assertion to fail - -- @tparam string message error message (may contain format string templates) - -- @param[opt] args any arguments required for formatting the message - assert: (condition, ...) => - args = table.pack ... - msg = table.remove args, 1 - unless condition - @assertFailed = true - @logger\logEx 1, msg, nil, nil, 0, unpack args - - - -- type assertions - - --- Fails the assertion if the specified value didn't have the expected type. - -- @param value the value to be type-checked - -- @tparam string expectedType the expected type - assertType: (val, expected) => - @checkArgTypes val: {val, "_any"}, expected: {expected, "string"} - actual = type val - @assert actual == expected, @@msgs.assert.type, expected, actual - - --- Fails the assertion if the types of the actual and expected value didn't match - -- @param actual the actual value - -- @param expected the expected value - assertSameType: (actual, expected) => - actualType, expectedType = type(actual), type expected - @assert actualType == expectedType, @@msgs.assert.sameType, expectedType, actualType - - --- Fails the assertion if the specified value isn't a boolean - -- @param value the value expected to be a boolean - assertBoolean: (val) => @assertType val, "boolean" - --- Shorthand for @{UnitTest:assertBoolean} - assertBool: (val) => @assertType val, "boolean" - - --- Fails the assertion if the specified value isn't a function - -- @param value the value expected to be a function - assertFunction: (val) => @assertType val, "function" - - --- Fails the assertion if the specified value isn't a number - -- @param value the value expected to be a number - assertNumber: (val) => @assertType val, "number" - - --- Fails the assertion if the specified value isn't a string - -- @param value the value expected to be a string - assertString: (val) => @assertType val, "string" - - --- Fails the assertion if the specified value isn't a table - -- @param value the value expected to be a table - assertTable: (val) => @assertType val, "table" - - --- Helper method to type-check arguments as a prerequisite to other asserts. - -- @local - -- @tparam {[string]={value, string}} args a hashtable of argument values and expected types - -- indexed by the respective argument names - checkArgTypes: (args) => - i, expected, actual = 1 - for name, types in pairs args - actual, expected = types[2], type types[1] - continue if expected == "_any" - @logger\assert actual == expected, @@msgs.assert.checkArgTypes, i, name, - expected, @format "type", types[1] - i += 1 - - - -- boolean asserts - - --- Fails the assertion if the specified value isn't the boolean `true`. - -- @param value the value expected to be `true` - assertTrue: (val) => - @assert val == true, @@msgs.assert.true, @format "type", val - - --- Fails the assertion if the specified value doesn't evaluate to boolean `true`. - -- In Lua this is only ever the case for `nil` and boolean `false`. - -- @param value the value expected to be truthy - assertTruthy: (val) => - @assert val, @@msgs.assert.truthy, @format "type", val - - --- Fails the assertion if the specified value isn't the boolean `false`. - -- @param value the value expected to be `false` - assertFalse: (val) => - @assert val == false, @@msgs.assert.false, @format "type", val - - --- Fails the assertion if the specified value doesn't evaluate to boolean `false`. - -- In Lua `nil` is the only other value that evaluates to `false`. - -- @param value the value expected to be falsy - assertFalsy: (val) => - @assert not val, @@msgs.assert.falsy, @format "type", val - - --- Fails the assertion if the specified value is not `nil`. - -- @param value the value expected to be `nil` - assertNil: (val) => - @assert val == nil, @@msgs.assert.nil, @format "type", val - - --- Fails the assertion if the specified value is `nil`. - -- @param value the value expected to not be `nil` - assertNotNil: (val) => - @assert val != nil, @@msgs.assert.notNil, @format "type", val - - - -- numerical asserts - - --- Fails the assertion if a number is out of the specified range. - -- @tparam number actual the number expected to be in range - -- @tparam number min the minimum (inclusive) value - -- @tparam number max the maximum (inclusive) value - assertInRange: (actual, min = -math.huge, max = math.huge) => - @checkArgTypes actual: {actual, "number"}, min: {min, "number"}, max: {max, "number"} - @assert actual >= min, @@msgs.assert.inRange, min, max, actual, "<", min - @assert actual <= max, @@msgs.assert.inRange, min, max, actual, ">", max - - --- Fails the assertion if a number is not lower than the specified value. - -- @tparam number actual the number to compare - -- @tparam number limit the lower limit (exclusive) - assertLessThan: (actual, limit) => - @checkArgTypes actual: {actual, "number"}, limit: {limit, "number"} - @assert actual < limit, @@msgs.assert.compare, "<", limit, actual - - --- Fails the assertion if a number is not lower than or equal to the specified value. - -- @tparam number actual the number to compare - -- @tparam number limit the lower limit (inclusive) - assertLessThanOrEquals: (actual, limit) => - @checkArgTypes actual: {actual, "number"}, limit: {limit, "number"} - @assert actual <= limit, @@msgs.assert.compare, "<=", limit, actual - - --- Fails the assertion if a number is not greater than the specified value. - -- @tparam number actual the number to compare - -- @tparam number limit the upper limit (exclusive) - assertGreaterThan: (actual, limit) => - @checkArgTypes actual: {actual, "number"}, limit: {limit, "number"} - @assert actual > limit, @@msgs.assert.compare, ">", limit, actual - - --- Fails the assertion if a number is not greater than or equal to the specified value. - -- @tparam number actual the number to compare - -- @tparam number limit the upper limit (inclusive) - assertGreaterThanOrEquals: (actual, limit) => - @checkArgTypes actual: {actual, "number"}, limit: {limit, "number"} - @assert actual >= limit, @@msgs.assert.compare, ">=", limit, actual - - --- Fails the assertion if a number is not in range of an expected value +/- a specified margin. - -- @tparam number actual the actual value - -- @tparam number expected the expected value - -- @tparam[opt=1e-8] number margin the maximum (inclusive) acceptable margin of error - assertAlmostEquals: (actual, expected, margin = 1e-8) => - @checkArgTypes actual: {actual, "number"}, min: {expected, "number"}, max: {margin, "number"} - - margin = math.abs margin - @assert math.abs(actual-expected) <= margin, @@msgs.assert.almostEquals, - expected, margin, actual - - --- Fails the assertion if a number differs from another value at most by a specified margin. - -- Inverse of @{assertAlmostEquals} - -- @tparam number actual the actual value - -- @tparam number value the value being compared against - -- @tparam[opt=1e-8] number margin the maximum (inclusive) margin of error for the numbers to be considered equal - assertNotAlmostEquals: (actual, value, margin = 1e-8) => - @checkArgTypes actual: {actual, "number"}, value: {value, "number"}, max: {margin, "number"} - - margin = math.abs margin - @assert math.abs(actual-value) > margin, @@msgs.assert.almostEquals, value, margin, actual - - --- Fails the assertion if a number is not equal to 0 (zero). - -- @tparam number actual the value - assertZero: (actual) => - @checkArgTypes actual: {actual, "number"} - @assert actual == 0, @@msgs.assert.zero, actual - - --- Fails the assertion if a number is equal to 0 (zero). - -- Inverse of @{assertZero} - -- @tparam number actual the value - assertNotZero: (actual) => - @checkArgTypes actual: {actual, "number"} - @assert actual != 0, @@msgs.assert.notZero - - --- Fails the assertion if a specified number has a fractional component. - -- All numbers in Lua share a common data type, which is usually a double, - -- which is the reason this is not a type check. - -- @tparam number actual the value - assertInteger: (actual) => - @checkArgTypes actual: {actual, "number"} - @assert math.floor(actual) == actual, @@msgs.assert.integer, actual - - --- Fails the assertion if a specified number is less than or equal 0. - -- @tparam number actual the value - -- @tparam[opt=false] boolean includeZero makes the assertion consider 0 to be positive - assertPositive: (actual, includeZero = false) => - @checkArgTypes actual: {actual, "number"}, includeZero: {includeZero, "boolean"} - res = includeZero and actual >= 0 or actual > 0 - @assert res, @@msgs.assert.positiveNegative, "positive", - includeZero and "included" or "excluded" - - --- Fails the assertion if a specified number is greater than or equal 0. - -- @tparam number actual the value - -- @tparam[opt=false] boolean includeZero makes the assertion not fail when a 0 is encountered - assertNegative: (actual, includeZero = false) => - @checkArgTypes actual: {actual, "number"}, includeZero: {includeZero, "boolean"} - res = includeZero and actual <= 0 or actual < 0 - @assert res, @@msgs.assert.positiveNegative, "positive", - includeZero and "included" or "excluded" - - - -- generic asserts - - --- Fails the assertion if a the actual value is not *equal* to the expected value. - -- On the requirements for equality see @{UnitTest:equals} - -- @param actual the actual value - -- @param expected the expected value - assertEquals: (actual, expected) => - @assert self.equals(actual, expected), @@msgs.assert.equals, type(actual), - @logger\dumpToString(actual), type(expected), @logger\dumpToString expected - - --- Fails the assertion if a the actual value is *equal* to the expected value. - -- Inverse of @{UnitTest:assertEquals} - -- @param actual the actual value - -- @param expected the expected value - assertNotEquals: (actual, expected) => - @assert not self.equals(actual, expected), @@msgs.assert.notEquals, - type(actual), @logger\dumpToString expected - - --- Fails the assertion if a the actual value is not *identical* to the expected value. - -- Uses the `==` operator, so in contrast to @{UnitTest:assertEquals}, - -- this assertion compares tables by reference. - -- @param actual the actual value - -- @param expected the expected value - assertIs: (actual, expected) => - @assert actual == expected, @@msgs.assert.is, @format("type", expected), - @format "type", actual - - --- Fails the assertion if a the actual value is *identical* to the expected value. - -- Inverse of @{UnitTest:assertIs} - -- @param actual the actual value - -- @param expected the expected value - assertIsNot: (actual, expected) => - @assert actual != expected, @@msgs.assert.isNot, @format "type", expected - - - -- table asserts - - --- Fails the assertion if the items of one table aren't *equal* to the items of another. - -- Unlike @{UnitTest:assertEquals} this ignores table keys, so e.g. two numerically-keyed tables - -- with equal items in a different order would still be considered equal. - -- By default this assertion only compares values at numerical indexes (see @{UnitTest:itemsEqual} for details). - -- @tparam table actual the first table - -- @tparam table expected the second table - -- @tparam[opt=true] boolean onlyNumericalKeys Disable this option to also compare items with non-numerical keys at the expense of a performance hit. - assertItemsEqual: (actual, expected, onlyNumKeys = true) => - @checkArgTypes { actual: {actual, "table"}, expected: {actual, "table"}, - onlyNumKeys: {onlyNumKeys, "boolean"} - } - - @assert self.itemsEqual(actual, expected, onlyNumKeys), - @@msgs.assert[onlyNumKeys and "itemsEqualNumericKeys" or "itemsEqualAllKeys"], - @logger\dumpToString(actual), @logger\dumpToString expected - - - --- Fails the assertion if the items of one table aren't *identical* to the items of another. - -- Like @{UnitTest:assertItemsEqual} this ignores table keys, however it compares table items by reference. - -- By default this assertion only compares values at numerical indexes (see @{UnitTest:itemsEqual} for details). - -- @tparam table actual the first table - -- @tparam table expected the second table - -- @tparam[opt=true] boolean onlyNumericalKeys Disable this option to also compare items with non-numerical keys - assertItemsAre: (actual, expected, onlyNumKeys = true) => - @checkArgTypes { actual: {actual, "table"}, expected: {actual, "table"}, - onlyNumKeys: {onlyNumKeys, "boolean"} - } - - @assert self.itemsEqual(actual, expected, onlyNumKeys, nil, true), - @@msgs.assert[onlyNumKeys and "itemsEqualNumericKeys" or "itemsEqualAllKeys"], - @logger\dumpToString(actual), @logger\dumpToString expected - - --- Fails the assertion if the numerically-keyed items of a table aren't continuous. - -- The rationale for this is that when iterating a table with ipairs or retrieving its length - -- with the # operator, Lua may stop processing the table once the item at index n is nil, - -- effectively hiding any subsequent values - -- @tparam table tbl the table to be checked - assertContinuous: (tbl) => - @checkArgTypes { tbl: {tbl, "table"} } - - realCnt, contCnt = 0, #tbl - for _, v in pairs tbl - if type(v) == "number" and math.floor(v) == v - realCnt += 1 - - @assert realCnt == contCnt, @@msgs.assert.continuous, contCnt+1, realCnt - - -- string asserts - - --- Fails the assertion if a string doesn't match the specified pattern. - -- Supports both Lua and Regex patterns. - -- @tparam string str the input string - -- @tparam string pattern the pattern to be matched against - -- @tparam[opt=false] boolean useRegex Enable this option to use Regex instead of Lua patterns - -- @tparam[optchain] re.Flags flags Any amount of regex flags as defined by the Aegisub re module - -- (see here for details: http://docs.aegisub.org/latest/Automation/Lua/Modules/re/#flags) - assertMatches: (str, pattern, useRegex = false, ...) => - @checkArgTypes { str: {str, "string"}, pattern: {pattern, "string"}, - useRegex: {useRegex, "boolean"} - } - - match = useRegex and re.match(str, pattern, ...) or str\match pattern, ... - @assert match, @@msgs.assert.matches, str, useRegex and "regex" or "Lua", pattern - - --- Fails the assertion if a string doesn't contain a specified substring. - -- Search is case-sensitive by default. - -- @tparam string str the input string - -- @tparam string needle the substring to be found - -- @tparam[opt=true] boolean caseSensitive Disable this option to use locale-dependent case-insensitive comparison. - -- @tparam[opt=1] number init the first byte to start the search at - assertContains: (str, needle, caseSensitive = true, init = 1) => - @checkArgTypes { str: {str, "string"}, needle: {needle, "string"}, - caseSensitive: {caseSensitive, "boolean"}, init: {init, "number"} - } - - _str, _needle = if caseSensitive - str\lower!, needle\lower! - else str, needle - @assert str\find(needle, init, true), str, needle, - caseSensitive and "sensitive" or "insensitive" - - -- function asserts - - - --- Fails the assertion if calling a function with the specified arguments doesn't cause it throw an error. - -- @tparam function func the function to be called - -- @param[opt] args any number of arguments to be passed into the function - assertError: (func, ...) => - @checkArgTypes { func: {func, "function"} } - - res = table.pack pcall func, ... - retCnt, success = res.n, table.remove res, 1 - res.n = nil - @assert success == false, @@msgs.assert.error, retCnt, @logger\dumpToString res - return res[1] - - --- Fails the assertion if a function call doesn't cause an error message that matches the specified pattern. - -- Supports both Lua and Regex patterns. - -- @tparam function func the function to be called - -- @tparam[opt={}] table args a table of any number of arguments to be passed into the function - -- @tparam string pattern the pattern to be matched against - -- @tparam[opt=false] boolean useRegex Enable this option to use Regex instead of Lua patterns - -- @tparam[optchain] re.Flags flags Any amount of regex flags as defined by the Aegisub re module - -- (see here for details: http://docs.aegisub.org/latest/Automation/Lua/Modules/re/#flags) - assertErrorMsgMatches: (func, params = {}, pattern, useRegex = false, ...) => - @checkArgTypes { func: {func, "function"}, params: {params, "table"}, - pattern: {pattern, "string"}, useRegex: {useRegex, "boolean"} - } - msg = @assertError func, unpack params - - match = useRegex and re.match(msg, pattern, ...) or msg\match pattern, ... - @assert match, @@msgs.assert.errorMsgMatches, msg, useRegex and "regex" or "Lua", pattern - - ---- A special case of the UnitTest class for a setup routine --- @classmod UnitTestSetup -class UnitTestSetup extends UnitTest - --- Runs the setup routine. - -- Only the @{UnitTestSetup} object is passed into the function. - -- Values returned by the setup routine are stored to be passed into the test functions later. - -- @treturn[1] boolean true (test succeeded) - -- @treturn[1] table retVals all values returned by the function packed into a table - -- @treturn[2] boolean false (test failed) - -- @treturn[2] string the error message describing how the test failed - run: => - @logger\logEx nil, @@msgs.run.setup, false - - res = table.pack pcall @f, @ - @success = table.remove res, 1 - @logResult res[1] - - if @success - @retVals = res - return true, @retVals - - return false, @errMsg - ---- A special case of the UnitTest class for a teardown routine --- @classmod UnitTestTeardown -class UnitTestTeardown extends UnitTest - --- Formats and writes a "running test x" message to the log. - -- @local - logStart: => - @logger\logEx nil, @@msgs.run.teardown, false - - ---- Holds a unit test class, i.e. a group of unit tests with common setup and teardown routines --- @classmod UnitTestClass -class UnitTestClass - msgs = { - run: { - runningTests: "Running test class '%s' (%d tests)..." - setupFailed: "Setup for test class '%s' FAILED, skipping tests." - abort: "Test class '%s' FAILED after %d tests, aborting." - testsFailed: "Done testing class '%s'. FAILED %d of %d tests." - success: "Test class '%s' completed successfully." - testNotFound: "Couldn't find requested test '%s'." - } - } - - --- Creates a new unit test class complete with a number of unit test as well as optional setup and teardown. - -- Instead of calling this constructor directly, it is recommended to call @{UnitTestSuite:new} instead, - -- which takes a table of test functions and creates test classes automatically. - -- @tparam string name a descriptive name for the test class - -- @tparam[opt={}] {[string] = function|table, ...} args a table of test functions by name; - -- indexes starting with "_" have special meaning and are not added as regular tests: - -- * _setup: a @{UnitTestSetup} routine - -- * _teardown: a @{UnitTestTeardown} routine - -- * _order: alternative syntax to the order parameter (see below) - -- @tparam [opt=nil (unordered)] {string, ...} A list of test names in the desired execution order. - -- Only tests mentioned in this table will be performed when running the whole test class. - -- If unspecified, all tests will be run in random order. - new: (@name, args = {}, @order, @testSuite) => - @logger = @testSuite.logger - @setup = UnitTestSetup "setup", args._setup, @ - @teardown = UnitTestTeardown "teardown", args._teardown, @ - @description = args._description - @order or= args._order - @tests = [UnitTest(name, f, @) for name, f in pairs args when "_" != name\sub 1,1] - - --- Runs all tests in the unit test class in the specified order. - -- @param[opt=false] abortOnFail stops testing once a test fails - -- @param[opt=(default)] overrides the default test order - -- @treturn[1] boolean true (test class succeeded) - -- @treturn[2] boolean false (test class failed) - -- @treturn[2] {@{UnitTest}, ...} a list of unit test that failed - run: (abortOnFail, order = @order) => - tests, failed = @tests, {} - if order - tests, mappings = {}, {test.name, test for test in *@tests} - for i, name in ipairs order - @logger\assert mappings[name], msgs.run.testNotFound, name - tests[i] = mappings[name] - testCnt, failedCnt = #tests, 0 - - @logger\log msgs.run.runningTests, @name, testCnt - @logger.indent += 1 - - success, res = @setup\run! - -- failing the setup always aborts - unless success - @logger.indent -= 1 - @logger\warn msgs.run.setupFailed, @name - return false, -1 - - for i, test in pairs tests - unless test\run unpack res - failedCnt += 1 - failed[#failed+1] = test - if abortOnFail - @logger.indent -= 1 - @logger\warn msgs.run.abort, @name, i - return false, failed - - @logger.indent -= 1 - @success = failedCnt == 0 - - if @success - @logger\log msgs.run.success, @name - return true - - @logger\log msgs.run.testsFailed, @name, failedCnt, testCnt - return false, failed - - ---- A DependencyControl unit test suite. --- Your test file/module must return a UnitTestSuite object in order to be recognized as a test suite. -class UnitTestSuite - msgs = { - run: { - running: "Running %d test classes for %s... " - aborted: "Aborting after %d test classes... " - classesFailed: "FAILED %d of %d test classes." - success: "All tests completed successfully." - classNotFound: "Couldn't find requested test class '%s'." - } - registerMacros: { - allDesc: "Runs the whole test suite." - } - new: { - badClassesType: "Test classes must be passed in either as a table or an import function, got a %s" - } - import: { - noTableReturned: "The test import function must return a table of test classes, got a %s." - } - } - - @UnitTest = UnitTest - @UnitTestClass = UnitTestClass - - --- Creates a complete unit test suite for a module or automation script. - -- Using this constructor will create all test classes and tests automatically. - -- @tparam string namespace the namespace of the module or automation script to test. - -- @tparam {[string] = table, ...}|function(self, dependencies, args...) args To create a UnitTest suite, - -- you must supply a hashtable of @{UnitTestClass} constructor tables by name. You can either do so directly, - -- or wrap it in a function that takes a number of arguments depending on how the tests are registered: - -- * self: the module being testsed (skipped for automation scripts) - -- * dependencies: a numerically keyed table of all the modules required by the tested script/module (in order) - -- * args: any additional arguments passed into the @{DependencyControl\registerTests} function. - -- Doing so is required to test automation scripts as well as module functions not exposed by its API. - -- indexes starting with "_" have special meaning and are not added as regular tests: - -- * _order: alternative syntax to the order parameter (see below) - -- @tparam [opt=nil (unordered)] {string, ...} An list of test class names in the desired execution order. - -- Only test classes mentioned in this table will be performed when running the whole test suite. - -- If unspecified, all test classes will be run in random order. - new: (@namespace, classes, @order) => - @logger = Logger defaultLevel: 3, fileBaseName: @namespace, fileSubName: "UnitTests", toFile: true - @classes = {} - switch type classes - when "table" then @addClasses classes - when "function" then @importFunc = classes - else @logger\error msgs.new.badClassesType, type classes - - --- Constructs test classes and adds them to the suite. - -- Use this if you need to add additional test classes to an existing @{UnitTestSuite} object. - -- @tparam {[string] = table, ...} args a hashtable of @{UnitTestClass} constructor tables by name. - addClasses: (classes) => - @classes[#@classes+1] = UnitTestClass(name, args, args._order, @) for name, args in pairs classes when "_" != name\sub 1,1 - if classes._order - @order or= {} - @order[#@order+1] = clsName for clsName in *classes._order - - --- Imports test classes from a function (passing in the specified arguments) and adds them to the suite. - -- Use this if you need to add additional test classes to an existing @{UnitTestSuite} object. - -- @tparam [opt] args a hashtable of @{UnitTestClass} constructor tables by name. - import: (...) => - return false unless @importFunc - classes = self.importFunc ... - @logger\assert type(classes) == "table", msgs.import.noTableReturned, type classes - @addClasses classes - @importFunc = nil - - --- Registers macros for running all or specific test classes of this suite. - -- If the test script is placed in the appropriate directory (according to module/automation script namespace), - -- this is automatically handled by DependencyControl. - registerMacros: => - menuItem = {"DependencyControl", "Run Tests", @name or @namespace, "[All]"} - aegisub.register_macro table.concat(menuItem, "/"), msgs.registerMacros.allDesc, -> @run! - for cls in *@classes - menuItem[4] = cls.name - aegisub.register_macro table.concat(menuItem, "/"), cls.description, -> cls\run! - - --- Runs all test classes of this suite in the specified order. - -- @param[opt=false] abortOnFail stops testing once a test fails - -- @param[opt=(default)] overrides the default test order - -- @treturn[1] boolean true (test class succeeded) - -- @treturn[2] boolean false (test class failed) - -- @treturn[2] {@{UnitTest}, ...} a list of unit test that failed - run: (abortOnFail, order = @order) => - classes, allFailed = @classes, {} - if order - classes, mappings = {}, {cls.name, cls for cls in *@classes} - for i, name in ipairs order - @logger\assert mappings[name], msgs.run.classNotFound, name - classes[i] = mappings[name] - - classCnt, failedCnt = #classes, 0 - @logger\log msgs.run.running, classCnt, @namespace - @logger.indent += 1 - - for i, cls in pairs classes - success, failed = cls\run abortOnFail - unless success - failedCnt += 1 - allFailed[#allFailed+1] = test for test in *failed - if abortOnFail - @logger.indent -= 1 - @logger\warn msgs.run.abort, i - return false, allFailed - - @logger.indent -= 1 - @success = failedCnt == 0 - if @success - @logger\log msgs.run.success - else @logger\log msgs.run.classesFailed, failedCnt, classCnt - - return @success, failedCnt > 0 and allFailed or nil \ No newline at end of file diff --git a/modules/DependencyControl/UpdateFeed.moon b/modules/DependencyControl/UpdateFeed.moon deleted file mode 100644 index 7ec8035..0000000 --- a/modules/DependencyControl/UpdateFeed.moon +++ /dev/null @@ -1,250 +0,0 @@ -json = require "json" -DownloadManager = require "DM.DownloadManager" - -DependencyControl = nil -Logger = require "l0.DependencyControl.Logger" -Common = require "l0.DependencyControl.Common" - -defaultLogger = Logger fileBaseName: "DepCtrl.UpdateFeed" - -class ScriptUpdateRecord extends Common - msgs = { - errors: { - noActiveChannel: "No active channel." - } - changelog: { - header: "Changelog for %s v%s (released %s):" - verTemplate: "v %s:" - msgTemplate: " • %s" - } - } - - new: (@namespace, @data, @config = {c:{}}, scriptType, autoChannel = true, @logger = defaultLogger) => - DependencyControl or= require "l0.DependencyControl" - @moduleName = scriptType == @@ScriptType.Module and @namespace - @[k] = v for k, v in pairs data - @setChannel! if autoChannel - - - getChannels: => - channels, default = {} - for name, channel in pairs @data.channels - channels[#channels+1] = name - if channel.default and not default - default = name - - return channels, default - - setChannel: (channelName = @config.c.activeChannel) => - with @config.c - .channels, default = @getChannels! - .lastChannel or= channelName or default - channelData = @data.channels[.lastChannel] - @activeChannel = .lastChannel - return false, @activeChannel unless channelData - @[k] = v for k, v in pairs channelData - - @files = @files and [file for file in *@files when not file.platform or file.platform == @@platform] or {} - return true, @activeChannel - - checkPlatform: => - @logger\assert @activeChannel, msgs.errors.noActiveChannel - return not @platforms or ({p,true for p in *@platforms})[@@platform], @@platform - - getChangelog: (versionRecord, minVer = 0) => - return "" unless "table" == type @changelog - maxVer = DependencyControl\parseVersion @version - minVer = DependencyControl\parseVersion minVer - - changelog = {} - for ver, entry in pairs @changelog - ver = DependencyControl\parseVersion ver - verStr = DependencyControl\getVersionString ver - if ver >= minVer and ver <= maxVer - changelog[#changelog+1] = {ver, verStr, entry} - - return "" if #changelog == 0 - table.sort changelog, (a,b) -> a[1]>b[1] - - msg = {msgs.changelog.header\format @name, DependencyControl\getVersionString(@version), @released or ""} - for chg in *changelog - chg[3] = {chg[3]} if type(chg[3]) ~= "table" - if #chg[3] > 0 - msg[#msg+1] = @logger\format msgs.changelog.verTemplate, 1, chg[2] - msg[#msg+1] = @logger\format(msgs.changelog.msgTemplate, 1, entry) for entry in *chg[3] - - return table.concat msg, "\n" - -class UpdateFeed extends Common - templateData = { - maxDepth: 7, - templates: { - feedName: {depth: 1, order: 1, key: "name" } - baseUrl: {depth: 1, order: 2, key: "baseUrl" } - feed: {depth: 1, order: 3, key: "knownFeeds", isHashTable: true } - namespace: {depth: 3, order: 1, parentKeys: {macros:true, modules:true} } - namespacePath: {depth: 3, order: 2, parentKeys: {macros:true, modules:true}, repl:"%.", to: "/" } - scriptName: {depth: 3, order: 3, key: "name" } - channel: {depth: 5, order: 1, parentKeys: {channels:true} } - version: {depth: 5, order: 2, key: "version" } - platform: {depth: 7, order: 1, key: "platform" } - fileName: {depth: 7, order: 2, key: "name" } - -- rolling templates - fileBaseUrl: {key: "fileBaseUrl", rolling: true } - } - sourceAt: {} - } - - msgs = { - trace: { - usingCached: "Using cached feed." - downloaded: "Downloaded feed to %s." - } - errors: { - downloadAdd: "Couldn't initiate download of %s to %s (%s)." - downloadFailed: "Download of feed %s to %s failed (%s)." - cantOpen: "Can't open downloaded feed for reading (%s)." - parse: "Error parsing feed." - } - } - - @defaultConfig = { - downloadPath: aegisub.decode_path "?temp/l0.#{@@__name}_feedCache" - dumpExpanded: false - } - @cache = {} - - fileBaseName = "l0.#{@@__name}_" - fileMatchTemplate = "l0.#{@@__name}_%x%x%x%x.*%.json" - feedsHaveBeenTrimmed = false - - -- precalculate some tables for the templater - templateData.rolling = {n, true for n,t in pairs templateData.templates when t.rolling} - templateData.sourceKeys = {t.key, t.depth for n,t in pairs templateData.templates when t.key} - with templateData - for i=1,.maxDepth - .sourceAt[i], j = {}, 1 - for name, tmpl in pairs .templates - if tmpl.depth==i and not tmpl.rolling - .sourceAt[i][j] = name - j += 1 - table.sort .sourceAt[i], (a,b) -> return .templates[a].order < .templates[b].order - - new: (@url, autoFetch = true, fileName, @config = {}, @logger = defaultLogger) => - DependencyControl or= require "l0.DependencyControl" - - -- fill in missing config values - @config[k] = v for k, v in pairs @@defaultConfig when @config[k] == nil - - -- delete old feeds - feedsHaveBeenTrimmed or= Logger(fileMatchTemplate: fileMatchTemplate, logDir: @config.downloadPath, maxFiles: 20)\trimFiles! - - @fileName = fileName or table.concat {@config.downloadPath, fileBaseName, "%04X"\format(math.random 0, 16^4-1), ".json"} - if @@cache[@url] - @logger\trace msgs.trace.usingCached - @data = @@cache[@url] - elseif autoFetch - @fetch! - - @downloadManager = DownloadManager aegisub.decode_path @config.downloadPath - - getKnownFeeds: => - return {} unless @data - return [url for _, url in pairs @data.knownFeeds] - -- TODO: maybe also search all requirements for feed URLs - - fetch: (fileName) => - @fileName = fileName if fileName - - dl, err = @downloadManager\addDownload @url, @fileName - unless dl - return false, msgs.errors.downloadAdd\format @url, @fileName, err - - @downloadManager\waitForFinish -> true - if dl.error - return false, msgs.errors.downloadFailed\format @url, @fileName, dl.error - - @logger\trace msgs.trace.downloaded, @fileName - - handle, err = io.open @fileName - unless handle - return false, msgs.errors.cantOpen\format err - - decoded, data = pcall json.decode, handle\read "*a" - unless decoded and data - -- luajson errors are useless dumps of whatever, no use to pass them on to the user - return false, msgs.errors.parse - - data[key] = {} for key in *{ @@ScriptType.name.legacy[@@ScriptType.Automation], - @@ScriptType.name.legacy[@@ScriptType.Module], - "knownFeeds"} when not data[key] - @data, @@cache[@url] = data, data - @expand! - return @data - - expand: => - {:templates, :maxDepth, :sourceAt, :rolling, :sourceKeys} = templateData - vars, rvars = {}, {i, {} for i=0, maxDepth} - - expandTemplates = (val, depth, rOff=0) -> - return switch type val - when "string" - val = val\gsub "@{(.-):(.-)}", (name, key) -> - if type(vars[name]) == "table" or type(rvars[depth+rOff]) == "table" - vars[name][key] or rvars[depth+rOff][name][key] - val\gsub "@{(.-)}", (name) -> vars[name] or rvars[depth+rOff][name] - when "table" - {k, expandTemplates v, depth, rOff for k, v in pairs val} - else val - - - recurse = (obj, depth = 1, parentKey = "", upKey = "") -> - -- collect regular template variables first - for name in *sourceAt[depth] - with templates[name] - if not .key - -- template variables are not expanded if they are keys - vars[name] = parentKey if .parentKeys[upKey] - elseif .key and obj[.key] - -- expand other templates used in template variable - obj[.key] = expandTemplates obj[.key], depth - vars[name] = obj[.key] - vars[name] = vars[name]\gsub(.repl, .to) if .repl - - -- update rolling template variables last - for name,_ in pairs rolling - rvars[depth][name] = obj[templates[name].key] or rvars[depth-1][name] or "" - rvars[depth][name] = expandTemplates rvars[depth][name], depth, -1 - obj[templates[name].key] and= rvars[depth][name] - - -- expand variables in non-template strings and recurse tables - for k,v in pairs obj - if sourceKeys[k] ~= depth and not rolling[k] - switch type v - when "string" - obj[k] = expandTemplates obj[k], depth - when "table" - recurse v, depth+1, k, parentKey - -- invalidate template variables created at depth+1 - vars[name] = nil for name in *sourceAt[depth+1] - rvars[depth+1] = {} - - recurse @data - - if @dumpExpanded - handle = io.open @fileName\gsub(".json$", ".exp.json"), "w" - handle\write(json.encode @data)\close! - - return @data - - getScript: (namespace, scriptType, config, autoChannel) => - section = @@ScriptType.name.legacy[scriptType] - scriptData = @data[section][namespace] - return false unless scriptData - ScriptUpdateRecord namespace, scriptData, config, scriptType, autoChannel, @logger - - getMacro: (namespace, config, autoChannel) => - @getScript namespace, false, config, autoChannel - - getModule: (namespace, config, autoChannel) => - @getScript namespace, true, config, autoChannel \ No newline at end of file diff --git a/modules/DependencyControl/Updater.moon b/modules/DependencyControl/Updater.moon deleted file mode 100644 index 888a105..0000000 --- a/modules/DependencyControl/Updater.moon +++ /dev/null @@ -1,524 +0,0 @@ -lfs = require "lfs" -DownloadManager = require "DM.DownloadManager" -PreciseTimer = require "PT.PreciseTimer" - -UpdateFeed = require "l0.DependencyControl.UpdateFeed" -fileOps = require "l0.DependencyControl.FileOps" -Logger = require "l0.DependencyControl.Logger" -Common = require "l0.DependencyControl.Common" -ModuleLoader = require "l0.DependencyControl.ModuleLoader" -DependencyControl = nil - -class UpdaterBase extends Common - @logger = Logger fileBaseName: "DependencyControl.Updater" - msgs = { - updateError: { - [0]: "Couldn't %s %s '%s' because of a paradox: module not found but updater says up-to-date (%s)" - [1]: "Couldn't %s %s '%s' because the updater is disabled." - [2]: "Skipping %s of %s '%s': namespace '%s' doesn't conform to rules." - [3]: "Skipping %s of unmanaged %s '%s'." - [4]: "No remaining feed available to %s %s '%s' from." - [6]: "The %s of %s '%s' failed because no suitable package could be found %s." - [5]: "Skipped %s of %s '%s': Another update initiated by %s is already running." - [7]: "Skipped %s of %s '%s': An internet connection is currently not available." - [10]: "Skipped %s of %s '%s': the update task is already running." - [15]: "Couldn't %s %s '%s' because its requirements could not be satisfied:" - [30]: "Couldn't %s %s '%s': failed to create temporary download directory %s" - [35]: "Aborted %s of %s '%s' because the feed contained a missing or malformed SHA-1 hash for file %s." - [50]: "Couldn't finish %s of %s '%s' because some files couldn't be moved to their target location:\n" - [55]: "%s of %s '%s' succeeded, couldn't be located by the module loader." - [56]: "%s of %s '%s' succeeded, but an error occured while loading the module:\n%s" - [57]: "%s of %s '%s' succeeded, but it's missing a version record." - [58]: "%s of unmanaged %s '%s' succeeded, but an error occured while creating a DependencyControl record: %s" - [100]: "Error (%d) in component %s during %s of %s '%s':\n— %s" - } - updaterErrorComponent: {"DownloadManager (adding download)", "DownloadManager"} - } - - getUpdaterErrorMsg: (code, name, scriptType, isInstall, detailMsg) => - if code <= -100 - -- Generic downstream error - return msgs.updateError[100]\format -code, msgs.updaterErrorComponent[math.floor(-code/100)], - @@terms.isInstall[isInstall], @@terms.scriptType.singular[scriptType], name, detailMsg - else - -- Updater error: - return msgs.updateError[-code]\format @@terms.isInstall[isInstall], - @@terms.scriptType.singular[scriptType], - name, detailMsg - -class UpdateTask extends UpdaterBase - dlm = DownloadManager! - msgs = { - checkFeed: { - downloadFailed: "Failed to download feed: %s" - noData: "The feed doesn't have any update information for %s '%s'." - badChannel: "The specified update channel '%s' wasn't present in the feed." - invalidVersion: "The feed contains an invalid version record for %s '%s' (channel: %s): %s." - unsupportedPlatform: "No download available for your platform '%s' (channel: %s)." - noFiles: "No files available to download for your platform '%s' (channel: %s)." - } - run: { - starting: "Starting %s of %s '%s'... " - fetching: "Trying to %sfetch missing %s '%s'..." - feedCandidates: "Trying %d candidate feeds (%s mode)..." - feedTrying: "Checking feed %d/%d (%s)..." - upToDate: "The %s '%s' is up-to-date (v%s)." - alreadyUpdated: "%s v%s has already been installed." - noFeedAvailExt: "(required: %s; installed: %s; available: %s)" - noUpdate: "Feed has no new update." - skippedOptional: "Skipped %s of optional dependency '%s': %s" - optionalNoFeed: "No feed available to download module from." - optionalNoUpdate: "No suitable download could be found %s." - } - - performUpdate: { - updateReqs: "Checking requirements..." - updateReady: "Update ready. Using temporary directory '%s'." - fileUnchanged: "Skipped unchanged file '%s'." - fileAddDownload: "Added Download %s ==> '%s'." - filesDownloading: "Downloading %d files..." - movingFiles: "Downloads complete. Now moving files to Aegisub automation directory '%s'..." - movedFile: "Moved '%s' ==> '%s'." - moveFileFailed: "Failed to move '%s' ==> '%s': %s" - updSuccess: "%s of %s '%s' (v%s) complete." - reloadNotice: "Please rescan your autoload directory for the changes to take effect." - unknownType: "Skipping file '%s': unknown type '%s'." - } - refreshRecord: { - unsetVirtual: "Update initated by another macro already fetched %s '%s', switching to update mode." - otherUpdate: "Update initated by another macro already updated %s '%s' to v%s." - } - } - - new: (@record, targetVersion = 0, @addFeeds, @exhaustive, @channel, @optional, @updater) => - DependencyControl or= require "l0.DependencyControl" - assert @record.__class == DependencyControl, "First parameter must be a #{DependencyControl.__name} object." - - @logger = @updater.logger - @triedFeeds = {} - @status = nil - @targetVersion = DependencyControl\parseVersion targetVersion - - -- set UpdateFeed settings - @feedConfig = { - downloadPath: aegisub.decode_path "?user/feedDump/" - dumpExpanded: true - } if @updater.config.c.dumpFeeds - - return nil, -1 unless @updater.config.c.updaterEnabled -- TODO: check if this even works - return nil, -2 unless @record\validateNamespace! - - set: (targetVersion, @addFeeds, @exhaustive, @channel, @optional) => - @targetVersion = DependencyControl\parseVersion targetVersion - return @ - - checkFeed: (feedUrl) => - -- get feed contents - feed = UpdateFeed feedUrl, false, nil, @feedConfig, @logger - unless feed.data -- no cached data available, perform download - success, err = feed\fetch! - unless success - return nil, msgs.checkFeed.downloadFailed\format err - - -- select our script and update channel - updateRecord = feed\getScript @record.namespace, @record.scriptType, @record.config, false - unless updateRecord - return nil, msgs.checkFeed.noData\format @@terms.scriptType.singular[@record.scriptType], @record.name - - success, currentChannel = updateRecord\setChannel @channel - unless success - return nil, msgs.checkFeed.badChannel\format currentChannel - - -- check if an update is available and satisfies our requirements - res, version = @record\checkVersion updateRecord.version - if res == nil - return nil, msgs.checkFeed.invalidVersion\format @@terms.scriptType.singular[@record.scriptType], - @record.name, currentChannel, tostring updateRecord.version - elseif res or @targetVersion > version - return false, nil, version - - -- check if our platform is supported/files are available to download - res, platform = updateRecord\checkPlatform! - unless res - return nil, msgs.checkFeed.unsupportedPlatform\format platform, currentChannel - if #updateRecord.files == 0 - return nil, msgs.checkFeed.noFiles\format platform, currentChannel - - return true, updateRecord, version - - - run: (waitLock, exhaustive = @updater.config.c.tryAllFeeds or @@exhaustive) => - logUpdateError = (code, extErr, virtual = @virtual) -> - if code < 0 - @logger\log @getUpdaterErrorMsg code, @record.name, @record.scriptType, virtual, extErr - return code, extErr - - with @record do @logger\log msgs.run.starting, @@terms.isInstall[.virtual], - @@terms.scriptType.singular[.scriptType], .name - - -- don't perform update of a script when another one is already running for the same script - return logUpdateError -10 if @running - - -- check if the script was already updated - if @updated and not exhaustive and @record\checkVersion @targetVersion - @logger\log msgs.run.alreadyUpdated, @record.name, DependencyControl\getVersionString @record.version - return 2 - - -- build feed list - userFeed, haveFeeds, feeds = @record.config.c.userFeed, {}, {} - if userFeed and not @triedFeeds[userFeed] - feeds[1] = userFeed - else - unless @triedFeeds[@record.feed] or haveFeeds[@record.feed] - feeds[1] = @record.feed - for feed in *@addFeeds - unless @triedFeeds[feed] or haveFeeds[feed] - feeds[#feeds+1] = feed - haveFeeds[feed] = true - - for feed in *@updater.config.c.extraFeeds - unless @triedFeeds[feed] or haveFeeds[feed] - feeds[#feeds+1] = feed - haveFeeds[feed] = true - - if #feeds == 0 - if @optional - @logger\log msgs.run.skippedOptional, @record.name, - @@terms.isInstall[@record.virtual], msgs.run.optionalNoFeed - return 3 - - return logUpdateError -4 - - -- check internet connection - return logUpdateError -7 unless dlm\isInternetConnected! - - -- get a lock on the updater - success, otherHost = @updater\getLock waitLock - return logUpdateError -5, otherHost unless success - - -- check feeds for update until we find and update or run out of feeds to check - -- normal mode: check feeds until an update matching the required version is found - -- exhaustive mode: check all feeds for updates and pick the highest version - - @logger\log msgs.run.feedCandidates, #feeds, exhaustive and "exhaustive" or "normal" - @logger.indent += 1 - - maxVer, updateRecord = 0 - for i, feed in ipairs feeds - @logger\log msgs.run.feedTrying, i, #feeds, feed - - res, rec, version = @checkFeed feed - @triedFeeds[feed] = true - if res == nil - @logger\log rec - elseif version > maxVer - maxVer = version - if res - updateRecord = rec - break unless exhaustive - else @logger\trace msgs.run.noUpdate - else - @logger\trace msgs.run.noUpdate - - @logger.indent -= 1 - - local code, res - wasVirtual = @record.virtual - unless updateRecord - -- for a script to be marked up-to-date it has to installed on the user's system - -- and the version must at least be that returned by at least one feed - if maxVer>0 and not @record.virtual and @targetVersion <= @record.version - @logger\log msgs.run.upToDate, @@terms.scriptType.singular[@record.scriptType], - @record.name, DependencyControl\getVersionString @record.version - return 0 - - res = msgs.run.noFeedAvailExt\format @targetVersion == 0 and "any" or DependencyControl\getVersionString(@targetVersion), - @record.virtual and "no" or DependencyControl\getVersionString(@record.version), - maxVer<1 and "none" or DependencyControl\getVersionString maxVer - - if @optional - @logger\log msgs.run.skippedOptional, @record.name, @@terms.isInstall[@record.virtual], - msgs.run.optionalNoUpdate\format res - return 3 - - return logUpdateError -6, res - - code, res = @performUpdate updateRecord - return logUpdateError code, res, wasVirtual - - performUpdate: (update) => - finish = (...) -> - @running = false - if @record.virtual or @record.recordType == @@RecordType.Unmanaged - ModuleLoader.removeDummyRef @record - return ... - - -- don't perform update of a script when another one is already running for the same script - return finish -10 if @running - @running = true - - -- set a dummy ref (which hasn't yet been set for virtual and unmanaged modules) - -- and record version to allow resolving circular dependencies - if @record.virtual or @record.recordType == @@RecordType.Unmanaged - ModuleLoader.createDummyRef @record - @record\setVersion update.version - - -- try to load required modules first to see if all dependencies are satisfied - -- this may trigger more updates - reqs = update.requiredModules - if reqs and #reqs > 0 - @logger\log msgs.performUpdate.updateReqs - @logger.indent += 1 - success, err = ModuleLoader.loadModules @record, reqs, {@record.feed} - @logger.indent -= 1 - unless success - @logger.indent += 1 - @logger\log err - @logger.indent -= 1 - return finish -15, err - - -- since circular dependencies are possible, our task may have completed in the meantime - -- so check again if we still need to update - return finish 2 if @updated and @record\checkVersion update.version - - - -- download updated scripts to temp directory - -- check hashes before download, only update changed files - - tmpDir = aegisub.decode_path "?temp/l0.#{DependencyControl.__name}_#{'%04X'\format math.random 0, 16^4-1}" - res, dir = fileOps.mkdir tmpDir - return finish -30, "#{tmpDir} (#{dir})" if res == nil - - @logger\log msgs.performUpdate.updateReady, tmpDir - - scriptSubDir = @record.namespace - scriptSubDir = scriptSubDir\gsub "%.","/" if @record.scriptType == @@ScriptType.Module - - dlm\clear! - for file in *update.files - file.type or= "script" - - baseName = scriptSubDir .. file.name - tmpName, prettyName = "#{tmpDir}/#{file.type}/#{baseName}", baseName - switch file.type - when "script" - file.fullName = "#{@record.automationDir}/#{baseName}" - when "test" - file.fullName = "#{@record.testDir}/#{baseName}" - prettyName ..= " (Unit Test)" - else - file.unknown = true - @logger\log msgs.performUpdate.unknownType, file.name, file.type - continue - continue if file.delete - - unless type(file.sha1)=="string" and #file.sha1 == 40 and tonumber(file.sha1, 16) - return finish -35, "#{prettyName} (#{tostring(file.sha1)\lower!})" - - if dlm\checkFileSHA1 file.fullName, file.sha1 - @logger\trace msgs.performUpdate.fileUnchanged, prettyName - continue - - dl, err = dlm\addDownload file.url, tmpName, file.sha1 - return finish -140, err unless dl - dl.targetFile = file.fullName - @logger\trace msgs.performUpdate.fileAddDownload, file.url, prettyName - - dlm\waitForFinish (progress) -> - @logger\progress progress, msgs.performUpdate.filesDownloading, #dlm.downloads - return true - @logger\progress! - - if #dlm.failedDownloads>0 - err = @logger\format ["#{dl.url}: #{dl.error}" for dl in *dlm.failedDownloads], 1 - return finish -245, err - - - -- move files to their destination directory and clean up - - @logger\log msgs.performUpdate.movingFiles, @record.automationDir - moveErrors = {} - @logger.indent += 1 - for dl in *dlm.downloads - res, err = fileOps.move dl.outfile, dl.targetFile, true - -- don't immediately error out if moving of a single file failed - -- try to move as many files as possible and let the user handle the rest - if res - @logger\trace msgs.performUpdate.movedFile, dl.outfile, dl.targetFile - else - @logger\log msgs.performUpdate.moveFileFailed, dl.outfile, dl.targetFile, err - moveErrors[#moveErrors+1] = err - @logger.indent -= 1 - - if #moveErrors>0 - return finish -50, @logger\format moveErrors, 1 - else lfs.rmdir tmpDir - os.remove file.fullName for file in *update.files when file.delete and not file.unknown - - -- Nuke old module refs and reload - oldVer, wasVirtual = @record.version, @record.virtual - - -- Update complete, refresh module information/configuration - if @record.scriptType == @@ScriptType.Module - ref = ModuleLoader.loadModule @record, @record, false, true - unless ref - if @record._error - return finish -56, @logger\format @record._error, 1 - else return finish -55 - - -- get a fresh version record - if type(ref.version) == "table" and ref.version.__class.__name == DependencyControl.__name - @record = ref.version - else - -- look for any compatible non-DepCtrl version records and create an unmanaged record - return finish -57 unless ref.version - success, rec = pcall DependencyControl, { moduleName: @record.moduleName, version: ref.version, - recordType: @@RecordType.Unmanaged, name: @record.name } - return finish -58, rec unless success - @record = rec - @ref = ref - - else with @record - .name, .version, .virtual = @record.name, DependencyControl\parseVersion update.version - @record\writeConfig! - - @updated = true - @logger\log msgs.performUpdate.updSuccess, @@terms.capitalize(@@terms.isInstall[wasVirtual]), - @@terms.scriptType.singular[@record.scriptType], - @record.name, DependencyControl\getVersionString @record.version - - -- Diplay changelog - @logger\log update\getChangelog @record, (DependencyControl\parseVersion oldVer) + 1 - @logger\log msgs.performUpdate.reloadNotice - - -- TODO: check handling of private module copies (need extra return value?) - return finish 1, DependencyControl\getVersionString @record.version - - - refreshRecord: => - with @record - wasVirtual, oldVersion = .virtual, .version - \loadConfig true - if wasVirtual and not .virtual or .version > oldVersion - @updated = true - @ref = ModuleLoader.loadModule @record, @record, false, true if .scriptType == @@ScriptType.Module - if wasVirtual - @logger\log msgs.refreshRecord.unsetVirtual, @@terms.scriptType.singular[.scriptType], .name - else - @logger\log msgs.refreshRecord.otherUpdate, @@terms.scriptType.singular[.scriptType], .name, - DependencyControl\getVersionString @record.version - -class Updater extends UpdaterBase - msgs = { - getLock: { - orphaned: "Ignoring orphaned in-progress update started by %s." - waitFinished: "Waited %d seconds." - abortWait: "Timeout reached after %d seconds." - waiting: "Waiting for update intiated by %s to finish..." - } - require: { - macroPassed: "%s is not a module." - upToDate: "Tried to require an update for up-to-date module '%s'." - } - scheduleUpdate: { - updaterDisabled: "Skipping update check for %s (Updater disabled)." - runningUpdate: "Running scheduled update for %s '%s'..." - } - } - new: (@host = script_namespace, @config, @logger = @@logger) => - @tasks = {scriptType, {} for _, scriptType in pairs @@ScriptType when "number" == type scriptType} - - addTask: (record, targetVersion, addFeeds = {}, exhaustive, channel, optional) => - DependencyControl or= require "l0.DependencyControl" - if record.__class != DependencyControl - depRec = {saveRecordToConfig: false, readGlobalScriptVars: false} - depRec[k] = v for k, v in pairs record - record = DependencyControl depRec - - task = @tasks[record.scriptType][record.namespace] - if task - return task\set targetVersion, addFeeds, exhaustive, channel, optional - else - task, err = UpdateTask record, targetVersion, addFeeds, exhaustive, channel, optional, @ - @tasks[record.scriptType][record.namespace] = task - return task, err - - require: (record, ...) => - @logger\assert record.scriptType == @@ScriptType.Module, msgs.require, record.name or record.namespace - @logger\log "%s module '%s'...", record.virtual and "Installing required" or "Updating outdated", record.name - task, code = @addTask record, ... - code, res = task\run true if task - - if code == 0 and not task.updated - -- usually we know in advance if a module is up to date so there's no reason to block other updaters - -- but we'll make sure to handle this case gracefully, anyway - @logger\debug msgs.require.upToDate, task.record.name or task.record.namespace - return ModuleLoader.loadModule task.record, task.record.namespace - elseif code >= 0 - return task.ref - else -- pass on update errors - return nil, code, res - - scheduleUpdate: (record) => - unless @config.c.updaterEnabled - @logger\trace msgs.scheduleUpdate.updaterDisabled, record.name or record.namespace - return -1 - - -- no regular updates for non-existing or unmanaged modules - if record.virtual or record.recordType == @@RecordType.Unmanaged - return -3 - - -- the update interval has not yet been passed since the last update check - if record.config.c.lastUpdateCheck and (record.config.c.lastUpdateCheck + @config.c.updateInterval > os.time!) - return false - - record.config.c.lastUpdateCheck = os.time! - record.config\write! - - task = @addTask record -- no need to check for errors, because we've already accounted for those case - @logger\trace msgs.scheduleUpdate.runningUpdate, @@terms.scriptType.singular[record.scriptType], record.name - return task\run! - - - getLock: (doWait, waitTimeout = @config.c.updateWaitTimeout) => - return true if @hasLock - - @config\load! - running, didWait = @config.c.updaterRunning - - if running and running.host != @host - if running.time + @config.c.updateOrphanTimeout < os.time! - @logger\log msgs.getLock.orphaned, running.host - elseif doWait - @logger\log msgs.getLock.waiting, running.host - timeout, didWait = waitTimeout, true - while running and timeout > 0 - PreciseTimer.sleep 1000 - timeout -= 1 - @config\load! - running = @config.c.updaterRunning - @logger\log timeout <= 0 and msgs.getLock.abortWait or msgs.getLock.waitFinished, - waitTimeout - timeout - - else return false, running.host - - -- register the running update in the config file to prevent collisions - -- with other scripts trying to update the same modules - -- TODO: store this flag in the db - - @config.c.updaterRunning = host: @host, time: os.time! - @config\write! - @hasLock = true - - -- reload important module version information from configuration - -- because another updater instance might have updated them in the meantime - if didWait - task\refreshRecord! for _,task in pairs @tasks[@@ScriptType.Module] - - return true - - releaseLock: => - return false unless @hasLock - @hasLock = false - @config.c.updaterRunning = false - @config\write! \ No newline at end of file diff --git a/modules/l0/AegisubShims.moon b/modules/l0/AegisubShims.moon new file mode 100644 index 0000000..d7bb826 --- /dev/null +++ b/modules/l0/AegisubShims.moon @@ -0,0 +1,9 @@ +aegisub = require "l0.AegisubShims.aegisub" + +-- Re-expose the shim's configuration hooks (see AegisubShims.aegisub) so callers can +-- relocate path tokens without reaching into the faux `aegisub` global. +return { + :aegisub + setPathToken: aegisub.__depCtrl.setPathToken + getPathToken: aegisub.__depCtrl.getPathToken +} diff --git a/modules/l0/AegisubShims/aegisub.moon b/modules/l0/AegisubShims/aegisub.moon new file mode 100644 index 0000000..b1539b3 --- /dev/null +++ b/modules/l0/AegisubShims/aegisub.moon @@ -0,0 +1,176 @@ +-- Headless shim for the Aegisub automation Lua API. +-- Installs `aegisub` as a global before any module that requires Aegisub-specific APIs. +-- +-- Configurable via environment variables: +-- DEPCTRL_USER_DIR — base for ?user / ?local (default: %APPDATA%\Aegisub / ~/.aegisub) +-- DEPCTRL_DATA_DIR — base for ?data (default: same as ?user; real Aegisub uses exe dir) +-- DEPCTRL_TEMP_DIR — base for ?temp (default: %TEMP% / /tmp) + +ffi = require "ffi" + +isWindows = ffi.os == "Windows" +pathSep = isWindows and "\\" or "/" + +tempDir = os.getenv("DEPCTRL_TEMP_DIR") or (isWindows and (os.getenv("TEMP")) or "/tmp") +userDir = os.getenv("DEPCTRL_USER_DIR") or + (isWindows and "#{os.getenv 'APPDATA'}\\Aegisub" or "#{os.getenv 'HOME'}/.aegisub") +dataDir = os.getenv("DEPCTRL_DATA_DIR") or userDir + +userPathsAddedToPackagePathLua = {} +userPathsAddedToPackagePathMoon = {} + +makePackagePaths = (dir, ext) -> {"#{dir}/?.#{ext}", "#{dir}/?/init.#{ext}"} + +-- Canonical token table matching libaegisub/path.cpp. +-- Empty string means "unset" — decode_path returns the path unchanged (same as real Aegisub). +-- ?audio, ?script, ?video are empty because no file is loaded headlessly. +pathTokens = { + "?audio": "" + "?data": dataDir + "?dictionary": dataDir .. pathSep .. "dictionaries" + "?local": userDir + "?script": "" + "?temp": tempDir + "?user": userDir + "?video": "" +} + +-- Sorted longest-first so ?dictionary matches before ?data. Rebuilt whenever a token +-- changes; decodePath closes over the `sortedTokens` upvalue, so reassigning it here is +-- enough to update the resolver. +local sortedTokens +rebuildSortedTokens = -> + sortedTokens = [{spec, dir} for spec, dir in pairs pathTokens] + table.sort sortedTokens, (a, b) -> #a[1] > #b[1] +rebuildSortedTokens! + +-- Normalize a token name to its canonical "?name" form so callers may pass either +-- "user" or "?user". +normalizeToken = (spec) -> + "string" == type(spec) and (spec\sub(1, 1) == "?" and spec or "?#{spec}") or spec + +---Points an Aegisub path token (e.g. "?user", "?temp") at a different directory. +---Lets headless callers relocate where DepCtrl reads/writes without environment variables. +---@param spec string The token to set, with or without the leading "?" ("user" or "?user"). +---@param dir? string The directory to resolve the token to; nil/"" marks it unset. +---@return string? dir The value the token now resolves to. +setPathToken = (spec, dir) -> + normalizedToken = normalizeToken spec + previousDir = pathTokens[normalizedToken] + return dir if previousDir == dir + + pathTokens[normalizedToken] = dir or "" + rebuildSortedTokens! + + if normalizedToken == "?user" + -- undo our previous additions to path list, add new ones that aren't already present, + -- and ensure the order of existing entries is unchanged to avoid messing up module shadowing + rebuildUserPaths = (pathStr, previouslyAdded, ext) -> + removed = {p, true for p in *previouslyAdded} + seen, ordered = {}, {} + for path in pathStr\gmatch "[^;]+" + continue if removed[path] or seen[path] + seen[path] = true + ordered[#ordered + 1] = path + + added = {} + for path in *makePackagePaths "#{dir}/automation/modules", ext + continue if seen[path] + seen[path] = true + ordered[#ordered + 1] = path + added[#added + 1] = path + + table.concat(ordered, ";"), added + + package.path, userPathsAddedToPackagePathLua = rebuildUserPaths package.path, userPathsAddedToPackagePathLua, "lua" + package.moonpath, userPathsAddedToPackagePathMoon = rebuildUserPaths package.moonpath, userPathsAddedToPackagePathMoon, "moon" + return dir + +---Returns the directory an Aegisub path token currently resolves to. +---@param spec string The token to query, with or without the leading "?". +---@return string? dir The configured directory, or nil if the token is unknown. +getPathToken = (spec) -> + dir = pathTokens[normalizeToken spec] + return dir if dir and dir != "" + +decodePath = (path) -> + for {spec, dir} in *sortedTokens + if path\sub(1, #spec) == spec + -- Empty dir means token is unset — return path as-is (Aegisub behavior). + return path if dir == "" + suffix = path\sub #spec + 1 + -- Consume the separator that follows the token, if any. + suffix = suffix\sub 2 if suffix\sub(1, 1) == "/" or suffix\sub(1, 1) == "\\" + return suffix == "" and dir or dir .. pathSep .. suffix + return path -- no token: return as-is + +aegisub = { + lua_automation_version: 4 + + decode_path: decodePath + + -- Always-nil stubs for context-dependent queries. + frame_from_ms: -> nil -- nil when no video loaded + ms_from_frame: -> nil + video_size: -> nil + keyframes: -> nil + get_audio_selection: -> nil + project_properties: -> nil + file_name: -> nil + + -- No-ops. + register_macro: -> nil + register_filter: -> nil + set_undo_point: -> nil + set_status_text: -> nil + + -- text_extents needs font rendering; error loudly rather than returning garbage. + text_extents: -> error "aegisub.text_extents is not available in headless mode", 2 + + gettext: (s) -> s + + cancel: -> error "aegisub.cancel", 2 + + -- These are normally injected by LuaProgressSink during macro execution. + -- We provide static stubs so scripts that call them at module load time don't crash. + -- The message is written verbatim (as real Aegisub does): callers append their own + -- newline, so writing one here would break same-line output and double-space the rest. + log: (level, msg, ...) -> + text = type(level) == "string" and level or msg + io.stderr\write tostring text or "" + + debug: { + out: (level, msg, ...) -> + text = type(level) == "string" and level or msg + io.stderr\write tostring text or "" + } + + progress: { + set: -> nil + task: -> nil + title: -> nil + is_cancelled: -> false + } + + dialog: { + display: -> {}, false + open: -> nil + save: -> nil + } + + clipboard: { + get: -> "" + set: -> true + } +} + +-- Shim-only configuration hooks, namespaced so they can't collide with the real +-- Aegisub API surface. Surfaced through l0.AegisubShims for callers to use. +aegisub.__depCtrl = { + :setPathToken + :getPathToken +} + +_G.aegisub = aegisub + +return aegisub diff --git a/modules/l0/DependencyControl.moon b/modules/l0/DependencyControl.moon new file mode 100644 index 0000000..4ba286f --- /dev/null +++ b/modules/l0/DependencyControl.moon @@ -0,0 +1,99 @@ +MIN_MOONSCRIPT_VERSION = "0.3.0" + +SemanticVersioning = require "l0.DependencyControl.SemanticVersioning" +moonscript = require 'moonscript.version' +assert SemanticVersioning\check(moonscript.version, MIN_MOONSCRIPT_VERSION), + [[ DependencyControl requires Moonscript v%s or later to work, +however the Version %s provided by your Aegisub installation is outdated. +Update to a recent Aegisub build to resolve this issue. +]]\format MIN_MOONSCRIPT_VERSION, moonscript.version + + +-- Install the module-provides searcher and register DepCtrl's bundled fallbacks before +-- the sub-modules below load. +ModuleProvider = require "l0.DependencyControl.ModuleProvider" +ModuleProvider\install! + +provideBundled = (providerName, aliases, forceVar) -> + if forceVar and os.getenv(forceVar) == "1" + impl = require providerName + package.loaded[alias] = impl for alias in *aliases + else + ModuleProvider\register alias, providerName for alias in *aliases + +provideBundled "l0.dkjson", {"json", "dkjson"} +provideBundled "l0.DependencyControl.shims.BadMutex", {"BM.BadMutex"}, "DEPCTRL_FORCE_BUILTIN_MUTEX" +provideBundled "l0.DependencyControl.shims.DownloadManager", {"DM.DownloadManager"}, "DEPCTRL_FORCE_BUILTIN_DOWNLOADER" +provideBundled "l0.DependencyControl.shims.PreciseTimer", {"PT.PreciseTimer"}, "DEPCTRL_FORCE_BUILTIN_TIMER" + +Common = require "l0.DependencyControl.Common" +ConfigHandler = require "l0.DependencyControl.ConfigHandler" +ConfigView = require "l0.DependencyControl.ConfigView" +Crypto = require "l0.DependencyControl.Crypto" +Downloader = require "l0.DependencyControl.Downloader" +Enum = require "l0.DependencyControl.Enum" +EventEmitter = require "l0.DependencyControl.EventEmitter" +FeedInventory = require "l0.DependencyControl.FeedInventory" +FeedLoader = require "l0.DependencyControl.FeedLoader" +FeedManager = require "l0.DependencyControl.FeedManager" +FeedTrust = require "l0.DependencyControl.FeedTrust" +FileOps = require "l0.DependencyControl.FileOps" +GitRepository = require "l0.DependencyControl.GitRepository" +Host = require "l0.DependencyControl.Host" +Lock = require "l0.DependencyControl.Lock" +Logger = require "l0.DependencyControl.Logger" +Record = require "l0.DependencyControl.Record" +Stub = require "l0.DependencyControl.Stub" +Timer = require "l0.DependencyControl.Timer" +UnitTestSuite = require "l0.DependencyControl.UnitTestSuite" +UpdateFeed = require "l0.DependencyControl.UpdateFeed" +Updater = require "l0.DependencyControl.Updater" + +---Main DependencyControl entry point. +---Provides package management and access to all sub-modules. +---@class DependencyControl: Record +class DependencyControl extends Record + @Common = Common + @ConfigHandler = ConfigHandler + @ConfigView = ConfigView + @Crypto = Crypto + @Downloader = Downloader + @Enum = Enum + @EventEmitter = EventEmitter + @FeedInventory = FeedInventory + @FeedLoader = FeedLoader + @FeedManager = FeedManager + @FeedTrust = FeedTrust + @FileOps = FileOps + @GitRepository = GitRepository + @Host = Host + @Lock = Lock + @Logger = Logger + @Record = Record + @Stub = Stub + @Timer = Timer + @UpdateFeed = UpdateFeed + @Updater = Updater + @UnitTestSuite = UnitTestSuite + @SemanticVersioning = SemanticVersioning + +rec = DependencyControl{ + name: "DependencyControl", + version: "0.7.0", + description: "Provides script management and auto-updating for Aegisub macros and modules.", + author: "line0", + url: "http://github.com/TypesettingTools/DependencyControl", + moduleName: "l0.DependencyControl", + feed: "https://raw.githubusercontent.com/TypesettingTools/DependencyControl/master/DependencyControl.json", + provides: { + {name: "BM.BadMutex", version: "^0.1.3"}, + {name: "DM.DownloadManager", version: "^0.3.1"}, + {name: "PT.PreciseTimer", version: "^0.1.6"}, + } +} +DependencyControl.__class.version = rec +LOADED_MODULES[rec.moduleName], package.loaded[rec.moduleName] = DependencyControl, DependencyControl +rec\requireModules! +rec\register DependencyControl + +return DependencyControl diff --git a/modules/l0/DependencyControl/Common.moon b/modules/l0/DependencyControl/Common.moon new file mode 100644 index 0000000..d26fb8a --- /dev/null +++ b/modules/l0/DependencyControl/Common.moon @@ -0,0 +1,351 @@ +ffi = require "ffi" +Crypto = require "l0.DependencyControl.Crypto" +Enum = require "l0.DependencyControl.Enum" +constants = require "l0.DependencyControl.Constants" + +---Serializes a value into a canonical string for hashing: table keys are emitted in sorted +---order so field ordering never affects the result, and every value is tagged with its type +---so distinct types can't collide (e.g. the number 1 vs. the string "1"). +---@param value any The value to canonicalize. +---@return string canonical The canonicalized string. +canonicalize = (value) -> + switch type value + when "table" + entries = {} + entries[#entries + 1] = "#{canonicalize k}=#{canonicalize v}" for k, v in pairs value + table.sort entries + "{#{table.concat entries, ","}}" + when "string" then "s:#{value}" + when "number" then "n:#{string.format "%.17g", value}" + when "boolean" then "b:#{value and 1 or 0}" + when "nil" then "nil" + else "#{type value}:#{tostring value}" + +-- Compares two values for deep equality. Tables are compared recursively; +-- other types use == except that two identical values always compare equal. +-- Circular references are handled. +_equals = (a, b, aType, bType) -> + treeA, treeB, depth = {}, {}, 0 + + recurse = (a, b, aType = type a, bType) -> + return true if a == b + bType or= type b + return false if aType != bType or aType != "table" + + return false if #a != #b + + aFieldCnt, bFieldCnt = 0, 0 + local tablesSeenAtKeys + + depth += 1 + treeA[depth], treeB[depth] = a, b + + for k, v in pairs a + vType = type v + if vType == "table" + tablesSeenAtKeys or= {} + tablesSeenAtKeys[k] = true + + for i = 1, depth + return true if v == treeA[i] and b[k] == treeB[i] + + unless recurse v, b[k], vType + depth -= 1 + return false + + aFieldCnt += 1 + + for k, v in pairs b + continue if tablesSeenAtKeys and tablesSeenAtKeys[k] + if bFieldCnt == aFieldCnt or not recurse v, a[k] + depth -= 1 + return false + bFieldCnt += 1 + + res = recurse getmetatable(a), getmetatable b + depth -= 1 + return res + + return recurse a, b, aType, bType + +-- Compares table items for equality ignoring keys. +-- Delegates table-vs-table comparisons to _equals. +_itemsEqual = (a, b, onlyNumKeys = true, ignoreExtraAItems, requireIdenticalItems) -> + seen, aTbls = {}, {} + aCnt, aTblCnt, bCnt = 0, 0, 0 + + findEqualTable = (bTbl) -> + for i, aTbl in ipairs aTbls + if _equals aTbl, bTbl + table.remove aTbls, i + seen[aTbl] = nil + return true + return false + + if onlyNumKeys + aCnt, bCnt = #a, #b + return false if not ignoreExtraAItems and aCnt != bCnt + + for v in *a + seen[v] = true + if "table" == type v + aTblCnt += 1 + aTbls[aTblCnt] = v + + for v in *b + if seen[v] + seen[v] = nil + continue + + if type(v) != "table" or requireIdenticalItems or not findEqualTable v + return false + + else + for _, v in pairs a + aCnt += 1 + seen[v] = true + if "table" == type v + aTblCnt += 1 + aTbls[aTblCnt] = v + + for _, v in pairs b + bCnt += 1 + if seen[v] + seen[v] = nil + continue + + if type(v) != "table" or requireIdenticalItems or not findEqualTable v + return false + + return false if not ignoreExtraAItems and aCnt != bCnt + + return true + + +getTableLength = (tbl) -> + n = 0 + n += 1 for _, _ in pairs tbl + return n + +isPureArrayTable = (tbl) -> + typ = type tbl + return false, nil, typ if typ != "table" + len = getTableLength tbl + return #tbl == len, len, typ + +---Flattens nested array tables into a single array up to the specified depth. Values that are not (or not converted to) pure array tables are included as-is. +---@param value any The value to flatten. +---@param depth? number Maximum depth to flatten (default 1). +---@param toArrayTable? fun(value: any, valueType: string): table?, boolean? Converts a non-array value to an array table. +---@return table flattened A flattened array table containing the flattened values. +---@return number flattenedCount The number of elements in the flattened array. +flatten = (value, depth = 1, toArrayTable) -> + flattened, f = {}, 0 + + recurse = (v, d) -> + isArray, _, typ = isPureArrayTable v + if toArrayTable and not isArray + v, isArray = toArrayTable v, typ + isArray = isPureArrayTable(v) if isArray == nil + if isArray and d > 0 + recurse nestedVal, d - 1 for nestedVal in *v + else + f += 1 + flattened[f] = v + + recurse value, depth + return flattened, f + + +---Shared constants, enums, and terminology used across DependencyControl modules. +---@class DependencyControlCommon +class DependencyControlCommon + msgs = { + validateNamespace: { + badNamespace: "Namespace '%s' failed validation. Namespace rules: must contain 1+ single dots, but not start or end with a dot; all other characters must be in [A-Za-z0-9-_]." + } + } + -- Some terms are shared across components + @platform = "#{ffi.os}-#{ffi.arch}" + + @moduleName = "l0.DependencyControl" + + ---@alias RecordType + ---| "managed" # Managed: a script/module DependencyControl installs and keeps up to date + ---| "unmanaged" # Unmanaged: a record describing a module DependencyControl tracks but does not update + RecordType = Enum "RecordType", { + Managed: "managed" + Unmanaged: "unmanaged" + } + @RecordType = RecordType + + ---@alias ScriptType + ---| "automation" # Automation: an automation script (macro / applied filter) + ---| "module" # Module: a require()-able module + ScriptType = Enum "ScriptType", { + Automation: "automation" + Module: "module" + } + @ScriptType = ScriptType + + ---@alias ScriptTypeSection + ---| "macros" # Automation scripts are stored in the "macros" section + ---| "modules" # Modules are stored in the "modules" section + ---Per-script type property names used in update feed data and config files. + @ScriptTypeSection = Enum "ScriptTypeSection", { + [ScriptType.Automation]: "macros" + [ScriptType.Module]: "modules" + } + + ---@alias FetchUntrustedFeeds + ---| "always" # Always: fetch untrusted feeds without asking (the default) + ---| "never" # Never: never fetch untrusted feeds + ---| "prompt" # Prompt: ask before fetching an untrusted feed; falls back to Never where no prompter is available (e.g. headless) + ---User policy for whether DependencyControl fetches feeds that are neither trusted nor blocked. + @FetchUntrustedFeeds = Enum "FetchUntrustedFeeds", { + Always: "always" + Never: "never" + Prompt: "prompt" + } + + @terms = { + scriptType: { + singular: { + [ScriptType.Automation]: "automation script" + [ScriptType.Module]: "module" + } + plural: { + [ScriptType.Automation]: "automation scripts" + [ScriptType.Module]: "modules" + } + } + + isInstall: { + [true]: "installation" + [false]: "update" + } + + capitalize: (str) -> (str\sub 1, 1)\upper! .. str\sub 2 + } + + ---Validates a DependencyControl namespace string. + ---@param namespace string + ---@return boolean? valid True when the namespace is well-formed. + ---@return string? err Validation error message when invalid. + @validateNamespace = (namespace) -> + segments = [seg for seg in namespace\gmatch "[^%.]+"] + _, dotCount = namespace\gsub "%.", "" + if #segments >= 2 and dotCount == #segments - 1 and not namespace\match "[^-._%w]" + return true + return false, msgs.validateNamespace.badNamespace\format namespace + + @getAutomationDir: (scriptType, rootDir = "?user") => + switch scriptType + when @ScriptType.Automation then aegisub.decode_path("#{rootDir}/automation/autoload") + when @ScriptType.Module then aegisub.decode_path("#{rootDir}/automation/include") + else nil + + @getTestDir = (scriptType, rootDir = "?user") => + switch scriptType + when @ScriptType.Automation then aegisub.decode_path("#{rootDir}/automation/tests/DepUnit/macros") + when @ScriptType.Module then aegisub.decode_path("#{rootDir}/automation/tests/DepUnit/modules") + else nil + + ---Whether DependencyControl is running headless — outside a real Aegisub session, on the Aegisub + ---shims (the CLI and unit test runner). Lets a script skip Aegisub-session-only startup work. + ---@return boolean headless + @isHeadless = -> aegisub[constants.DEPCTRL_PRIVATE_GLOBAL_VAR_PREFIX] != nil + + + ---Deep equality comparison. Tables compared recursively; other types use ==. + ---Circular references are handled. Metatables are included in the comparison. + ---@param a any + ---@param b any + ---@return boolean equal + @equals = _equals + + ---Compares table items for equality, ignoring keys. + ---By default only numerical indexes are compared. + ---@param a table + ---@param b table + ---@param onlyNumKeys? boolean Compare only sequential numeric indices (default true). + ---@param ignoreExtraAItems? boolean Allow `a` to contain items absent from `b` (default false). + ---@param requireIdenticalItems? boolean Require identical (not merely equal) table items (default false). + ---@return boolean equal + @itemsEqual = _itemsEqual + + ---Shallow-copies a table (no metatable). + ---@param tbl table The table to copy. + ---@return table copy The copied table. + @copy = (tbl) -> {k, v for k, v in pairs tbl} + + ---Deep-copies a table recursively (no metatables). + ---@param tbl table The table to deep-copy. + ---@return table copy The deep-copied table. + deepCopy = (tbl) -> {k, (type(v) == "table" and deepCopy(v) or v) for k, v in pairs tbl} + + ---Deep-copies a table recursively (no metatables). + ---@param tbl table The table to deep-copy. + ---@return table copy The deep-copied table. + @deepCopy = deepCopy + + ---Builds (or extends) a set from an array's values: each value becomes a key mapped to `value`. + ---@param source any[] Array whose values become the set's keys. + ---@param target? table Table to populate (default a new table). + ---@param overwrite? boolean Overwrite keys already present in `target` (default true). + ---@param value? any Value to map each key to (default true). + ---@return table set The populated `target`. + @makeSet = (source, target = {}, overwrite = true, value = true) -> + target[v] = value for v in *source when overwrite or not target[v] + return target + + ---Reports whether an array contains a value (compared with `==`). + ---@param list any[] The array to search. + ---@param value any The value to look for. + ---@return boolean included Whether `value` is an element of `list`. + @listIncludes = (list, value) -> + for entry in *list + return true if entry == value + return false + + ---Fills in missing entries of `tbl` from `defaults`, mutating `tbl` in place. + ---@param tbl table The table to fill in. + ---@param defaults table Default key/value pairs. + ---@param predicate? fun(value: any, key: any, tbl: table): boolean Per-key test for whether to apply the default; when omitted, defaults apply wherever `tbl[key]` is nil. + ---@return number addedCount The number of defaults applied. + @addDefaults = (tbl, defaults, predicate) -> + addedCnt = 0 + for k, v in pairs defaults + if not predicate and tbl[k] == nil or predicate and predicate tbl[k], k, tbl + addedCnt += 1 + tbl[k] = v + return addedCnt + + ---Strips leading and trailing whitespace from a string. + ---@param str string The string to trim. + ---@return string trimmed The trimmed string. + @trim = (str) -> (str\gsub "^%s*(.-)%s*$", "%1") + + ---Flattens nested array tables into a single array up to the specified depth. Values that are not (or not converted to) pure array tables are included as-is. + ---@param value any The value to flatten. + ---@param depth? number Maximum depth to flatten (default 1). + ---@param toArrayTable? fun(value: any, valueType: string): table?, boolean? Converts a non-array value to an array table. + ---@return table flattened A flattened array table containing the flattened values. + ---@return number flattenedCount The number of elements in the flattened array. + @flatten = flatten + + ---Produces a deterministic SHA-1 hash of a (possibly nested) Lua value. + ---Table keys are sorted before hashing, so field ordering never affects the result; pass an + ---object pruned to just the fields you care about to obtain a stable content signature that + ---ignores irrelevant differences. Useful for cheaply detecting whether semantic content changed. + ---@param value any The value to hash. + ---@return string hash A 40-character lowercase SHA-1 hex digest. + @getObjectHash = (value) -> Crypto.sha1 canonicalize value + + ---Generates a random RFC-4122 version-4 UUID string. + ---@return string uuid + @uuid = -> + -- https://gist.github.com/jrus/3197011 + "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx"\gsub "[xy]", (c) -> + v = c == "x" and math.random(0, 0xf) or math.random 8, 0xb + return "%x"\format v diff --git a/modules/l0/DependencyControl/ConfigHandler.moon b/modules/l0/DependencyControl/ConfigHandler.moon new file mode 100644 index 0000000..325bb85 --- /dev/null +++ b/modules/l0/DependencyControl/ConfigHandler.moon @@ -0,0 +1,462 @@ +json = require "json" +constants = require "l0.DependencyControl.Constants" +fileOps = require "l0.DependencyControl.FileOps" +Logger = require "l0.DependencyControl.Logger" +Lock = require "l0.DependencyControl.Lock" +ConfigView = require "l0.DependencyControl.ConfigView" +Common = require "l0.DependencyControl.Common" +JsonSchema = require "l0.DependencyControl.JsonSchema" + +---JSON-backed configuration manager with cooperative cross-script locking. +---Manages one JSON file per instance. Use ConfigView (via getView or ConfigView.get) +---to access specific hives (nested sections) of the config. +---@class ConfigHandler +class ConfigHandler + msgs = { + get: { + failedLoad: "Could not provide a ConfigHandler because there was an issue loading the configuration file: %s" + failedCreate: "Failed to create ConfigHandler for file '%s': %s" + } + getHive: { + unexpected: "An unexpected error occurred while trying to create hive '%s' on ConfigHandler for file '%s'" + } + __getOverlappingViews: { + differentHandler: "Other view on config file '%s' does not belong to this config handler of config file '%s'." + } + getView: { + failedView: "Failed to get #{ConfigView.__name} '%s' on ConfigHandler for file '%s': %s" + failedHandler: "Failed to get ConfigHandler for file '%s' while trying to acquire a view on #{ConfigView.__name}: %s" + } + mergeHive: { + badKey: "Can't merge hive because the path key #%d (%s) points to a %s." + } + new: { + badPath: "Couldn't validate specified config file path '%s': %s" + failedLoad: "Failed to load config file '%s': %s" + } + readFile: { + failedLock: "Failed to lock config file for reading: %s" + fileNotFound: "Couldn't find config file '%s'." + jsonDecodeError: "JSON parse error: %s" + configCorrupted: [[An error occurred while parsing the JSON config file. +A backup of the corrupted configuration has been written to '%s'. +Reload your automation scripts to generate a new configuration file.]] + failedHandle: "Failed to acquire a handle for reading the config file: %s" + badJsonRoot: "JSON root element must be an array or a hashtable, got a %s." + } + load: { + noFilePath: "Can't load because no config file is set." + noFile: "Starting with a fresh config because the config file '%s' is missing (%s)..." + migrationSaveFailed: "Migrated config '%s' to the current schema, but couldn't save it (%s); will retry on next load." + } + save: { + failedWhole: "Failed to save complete config to file '%s': %s" + failedHives: "Failed to save hives %s into config file '%s': %s" + failedMerge: "Failed to merge config hive %s into file '%s': %s" + failedClean: "Failed to clean config hive %s in file '%s': %s" + failedLock: "Failed to lock config file for saving: %s" + failedRead: "Failed to read config file '%s': %s." + noFile: "Can't save because no config file is set." + fileCreate: "Config file '%s' doesn't exist, will write a fresh one..." + } + traverseHive: { + badKey: "Can't retrieve hive because the path key #%d (%s) points to a %s." + } + writeFile: { + writing: "Writing config file '%s'..." + failedLock: "Failed to lock config file for writing: %s" + failedSerialize: "Failed to serialize configuration to JSON: %s" + failedHandle: "Failed to acquire a handle for writing the config file: %s" + } + } + + -- make references to provided handlers weak to allow for gc + @handlers = setmetatable {}, {__mode: 'v'} + @logger = Logger fileBaseName: "#{constants.DEPCTRL_SHORT_NAME}.#{@__name}", fileSubName: script_namespace + + ---Returns an existing handler for filePath, or creates and optionally loads one. + ---@param filePath string + ---@param logger? Logger + ---@param noLoad? boolean Don't load the file immediately (default false). + ---@param schemaOpts? { schemaId: string, migrate: fun(config: table, current?: string, target: string): boolean } Schema id this handler targets and the migration callback run when a loaded file's `$schema` differs. Applied on first creation; a cached handler keeps the opts it was created with. + ---@return ConfigHandler? handler + ---@return string? err + @get = (filePath, logger = @logger, noLoad = false, schemaOpts) => + return handler for path, handler in pairs @@handlers when path == filePath + + path, msg = fileOps.validateFullPath filePath, true + return nil, msgs.new.badPath\format filePath, msg unless path + + success, handler = pcall ConfigHandler, path, logger, schemaOpts + unless success + return nil, msgs.get.failedCreate\format filePath, handler + + @@handlers[path] = handler + + unless noLoad + success, msg = handler\load! + return nil, msgs.get.failedLoad\format filePath, msg unless success + + return handler + + + ---Returns a ConfigView for the given file and hive path, creating a handler if needed. + ---@param filePath string + ---@param hivePath string|string[] + ---@param defaults? table Default values for the hive. + ---@param logger? Logger + ---@return ConfigView? view + ---@return string? err + @getView = (filePath, hivePath, defaults, logger) => + handler, msg = @get filePath, logger + return nil, msgs.getView.failedHandler\format filePath, msg unless handler + + return handler\getView hivePath, defaults + + + ---Creates a ConfigHandler for the given file. Does not load from disk. + ---@param filePath? string + ---@param logger? Logger + ---@param schemaOpts? { schemaId: string, migrate: fun(config: table, current?: string, target: string): boolean } The `$schema` this handler targets and the migration callback invoked on load when a file's `$schema` differs. + new: (filePath, @logger = Logger(fileBaseName: @@__name), schemaOpts = {}) => + @views = setmetatable {}, {__mode: 'k'} + @config = {} + -- the loaded file's `$schema`, exposed so views can see which schema their values conform to + @schemaId = nil + @__targetSchemaId = schemaOpts.schemaId + @__migrate = schemaOpts.migrate + if filePath + path, msg = fileOps.validateFullPath filePath, true + @logger\assert path, msgs.new.badPath, filePath, msg + @filePath = path + -- config files are shared across concurrent Aegisub instances, so the lock + -- must exclude across processes, not just within this one + @lock = Lock { + namespace: "l0.DependencyControl.ConfigHandler", resource: @filePath + holderName: @@__name, logger: @logger, scope: Lock.Scope.Global + } + + + readFile = (waitLockTime, useLock = true) => + mode, file = fileOps.attributes @filePath, "mode" + if mode == nil + return nil, file + + elseif not mode + @logger\trace msgs.readFile.fileNotFound, @filePath + return false, msgs.readFile.fileNotFound\format @filePath + + if useLock + lockState, msg = @lock\lock waitLockTime + if lockState != Lock.LockState.Held + return nil, msgs.readFile.failedLock\format msg + + handle, msg = io.open file, "r" + unless handle + @lock\release! if useLock + return nil, msgs.readFile.failedHandle\format msg + + data = handle\read "*a" + handle\close! + + @lock\release! if useLock + + success, res = pcall json.decode, data + unless success + -- JSON parse error usually points to a corrupted config file + -- Rename the broken file to allow generating a new one + -- so the user can continue their work + @logger\debug msgs.readFile.jsonDecodeError, res + backup = @filePath .. ".corrupted" + fileOps.copy @filePath, backup + fileOps.remove @filePath, false, true + + @logger\warn msgs.readFile.configCorrupted, backup + return false, msgs.readFile.configCorrupted\format backup + + if "table" != type res + return nil, msgs.readFile.badJsonRoot\format type res + + return res + + + writeFile = (config, waitLockTime, haveLock = false) => + success, res = pcall json.encode, ConfigHandler\getSerializableCopy config + unless success + return nil, msgs.writeFile.failedSerialize\format res + + unless haveLock + lockState, msg = @lock\lock waitLockTime + if lockState != Lock.LockState.Held + return nil, msgs.writeFile.failedLock\format msg + + handle, msg = io.open(@filePath, "w") + unless handle + @lock\release! unless haveLock + return nil, msgs.writeFile.failedHandle\format msg + + @logger\trace msgs.writeFile.writing, @filePath + handle\setvbuf "full", 10e6 + handle\write res + handle\flush! + handle\close! + + @lock\release! unless haveLock + return true + + + hasNonPrivateFields = (tbl) -> + for k, _ in pairs tbl + if k\sub(1, 1) == "_" + continue + else return true + + return false + + + makeHive = (path, config) -> + return config if #path == 0 + recurse = (path, hive, depth, config) -> + return if depth > #path + hive[path[depth]] = depth == #path and config or {} + return recurse path, hive[path[depth]], depth + 1, config + + hive = {} + recurse path, hive, 1, config + return hive + + + traverseHive = (path, config, depth = #path) -> + for i, key in ipairs path + break if i > depth + switch type config + when "nil" + return false + when "table" + config = config[key] + else + return nil, msgs.traverseHive.badKey\format i, key, type config + + return config or false + + + mergeHive = (path, source, target, depth = 1) -> + -- merging in a root hive overwrites target with source + if #path == 0 + target[k] = nil for k, _ in pairs target + target[k] = source[k] for k, _ in pairs source + return true + + key = path[depth] + + if depth == #path + target[key] = source[key] + return true + + if target[key] != nil and "table" != type target[key] + return nil, msgs.mergeHive.badKey\format depth, key, type target[key] + + target[key] or= {} + return mergeHive path, source[key], target[key], depth + 1 + + + purgeHive = (path, config) -> + if #path == 0 + config[k] = nil for k, _ in pairs config + + for i = #path, 1, -1 + parent, msg = traverseHive path, config, i-1 + switch parent + when nil then return nil, msg + when false then continue + + parent[path[i]] = nil + break if hasNonPrivateFields parent + + return true + + + cleanHive = (path, config) -> + hive, msg = traverseHive path, config + return hive, msg if hive == nil + return true if hive == false -- path absent in file config; nothing to purge + + return false if hasNonPrivateFields hive + return purgeHive path, config + + + -- copied from Aegisub util.moon, adjusted to skip private keys + ---Deep-copies a value while skipping private keys prefixed with "_". + ---@param val any + ---@return any copy + @getSerializableCopy = (val) => + seen = {} + copy = (val) -> + return val if type(val) != 'table' + return {} if seen[val] -- nuke circular references which JSON doesn't support + seen[val] = val + {k, copy(v) for k, v in pairs val when type(k) != "string" or k\sub(1,1) != "_"} + copy val + + + ---Returns the config table at the given hive path, creating it if missing. + ---@param path string[] + ---@return table? hive + ---@return string? err + getHive: (path) => + hive, msg = traverseHive path, @config + switch hive + when nil + return nil, msg + when false + res, msg = mergeHive path, makeHive(path), @config + return nil, msg unless res + + hive, msg = traverseHive path, @config + unless hive + @logger\warn msgs.getHive.unexpected, path, @filePath + return nil, msgs.getHive.unexpected\format path, @filePath + + return hive + + + ---Returns views on the same handler whose hive paths overlap with targetView. + ---@param targetView ConfigView + ---@return ConfigView[]? views nil when targetView belongs to a different handler. + ---@return string? err + ---@private + __getOverlappingViews: (targetView) => + if targetView.__configHandler != @ + return nil, msgs.__getOverlappingViews.differentHandler\format targetView.__configHandler.filePath, @filePath + + return for view, _ in pairs @views + continue if view == targetView or not targetView\isOverlappingView view + view + + + ---Creates and registers a ConfigView for the given hive path. + ---@param hivePath string|string[] + ---@param defaults? table Default values for the hive. + ---@return ConfigView? view + ---@return string? err + getView: (hivePath, defaults) => + success, view = pcall ConfigView, @, hivePath, defaults + + unless success + return nil, msgs.getView.failedView\format hivePath, @filePath, view + + @views[view] = true + return view + + + ---Reads the config file and refreshes the in-memory config and all (or specified) views. + ---@param views? ConfigView|ConfigView[] Views to refresh (default: all registered views). + ---@param waitLockTime? number Seconds to wait for the config lock. + ---@return boolean? success + ---@return string? err + load: (views, waitLockTime) => + return nil, msgs.load.noFilePath unless @filePath + if type(views) == "table" and views.__class == ConfigView + views = {views} + + config, msg = readFile @, waitLockTime + return nil, msg if config == nil + + @logger\debug msgs.load.noFile, @filePath, msg unless config + -- config file may not yet exist or have been reset due to corruption + config or= {} + + if views == nil or @config == nil + -- bring a pre-target config up to the handler's schema, then persist the one-time change + migrated = false + if @__migrate and @__targetSchemaId + currentSchema = config[JsonSchema.JSON_SCHEMA_ID_KEYWORD] + if currentSchema != @__targetSchemaId and @.__migrate config, currentSchema, @__targetSchemaId + config[JsonSchema.JSON_SCHEMA_ID_KEYWORD] = @__targetSchemaId + migrated = true + @schemaId = config[JsonSchema.JSON_SCHEMA_ID_KEYWORD] + + @config = config + view\refresh! for view, _ in pairs @views + + if migrated + ok, msg = @save! + @logger\warn msgs.load.migrationSaveFailed, @filePath, msg unless ok + return true + + viewsToRefresh = Common.makeSet views + + for view in *views + hiveConfig, msg = traverseHive view.__hivePath, config + switch hiveConfig + when nil + return nil, msg + when false + mergeHive view.__hivePath, makeHive(view.__hivePath), @config + else mergeHive view.__hivePath, makeHive(view.__hivePath, hiveConfig), @config + + Common.makeSet @__getOverlappingViews(view), viewsToRefresh, false + + view\refresh! for view, _ in pairs viewsToRefresh + + return true + + + ---Writes the config file, merging only the specified views (or the full config if nil). + ---@param views? ConfigView|ConfigView[] Views to merge (default: the whole config). + ---@param waitLockTime? number Seconds to wait for the config lock. + ---@return boolean? success + ---@return string? err + save: (views, waitLockTime) => + return nil, msgs.save.noFile unless @filePath + if type(views) == "table" and views.__class == ConfigView + views = {views} + + -- get a lock to avoid concurrent config file access + lockState, msg = @lock\lock waitLockTime + if lockState != Lock.LockState.Held + return nil, msgs.save.failedLock\format msg + + -- read the config file under the lock we already hold (useLock false, or readFile would release it) + config, err = readFile @, waitLockTime, false + if config == nil + @lock\release! + return nil, msgs.save.failedRead\format @filePath, err + + @logger\trace msgs.save.fileCreate, @filePath unless config + config or= {} + + -- save the whole config file if desired + if views == nil + success, msg = writeFile @, @config, nil, true + @lock\release! + return if success + true + else nil, msgs.save.failedWhole\format @filePath, msg + + -- otherwise only merge in the specified views + for view in *views + success, msg = mergeHive view.__hivePath, @config, config + unless success + @lock\release! + return nil, msgs.save.failedMerge\format view.__hivePath, @filePath, msg + + success, msg = cleanHive view.__hivePath, config + if success == nil + @lock\release! + return nil, msgs.save.failedClean\format view.__hivePath, @filePath, msg + + success, msg = writeFile @, config, nil, true + @lock\release! + return if success + true + else nil, msgs.save.failedHives\format views, @filePath, msg + + + ---Removes a view's hive from the in-memory config and returns the fresh (empty) hive. + ---@param hive ConfigView + ---@return table? hive + ---@return string? err + purgeHive: (hive) => + purgeHive hive.__hivePath, @config + return @getHive hive.__hivePath diff --git a/modules/l0/DependencyControl/ConfigView.moon b/modules/l0/DependencyControl/ConfigView.moon new file mode 100644 index 0000000..2011e87 --- /dev/null +++ b/modules/l0/DependencyControl/ConfigView.moon @@ -0,0 +1,243 @@ +Common = require "l0.DependencyControl.Common" +local ConfigHandler + +-- A read/write view over a user section that holds only some of its keys, falling through per key to the +-- section default: a user-set value wins, an absent key reads the default. Needed because the top-level +-- default fallback is whole-section, so a partially-populated section (e.g. what the flat->sectioned config +-- migration leaves behind, `updates = {enabled}`) would otherwise read nil for its unset sibling keys. +-- Writes go straight to the user section, so defaults are never materialized into stored config. +mergeSection = (userSection, defaultSection) -> + setmetatable {}, { + __index: (_, k) -> if userSection[k] != nil then userSection[k] else defaultSection[k] + __newindex: (_, k, v) -> userSection[k] = v + __len: -> 0 + __ipairs: -> error "numerically indexed config hive keys are not supported" + __pairs: -> + merged = {} + merged[k] = v for k, v in pairs defaultSection + merged[k] = v for k, v in pairs userSection + return next, merged + } + +---A view into a hive (nested path) of a ConfigHandler's JSON config file. +---Holds the proxy/defaults machinery and exposes @c / @config / @userConfig. +---Multiple views on the same file are coordinated through their shared ConfigHandler. +---@class ConfigView +class ConfigView + msgs = { + new: { + failedRetrieveHive: "Failed to retrieve hive %s from ConfigHandler: %s" + } + isOverlappingView: { + differentHandler: "Other view on config file '%s' does not belong to the same config handler as this view on config file '%s'." + } + } + + ---Returns a ConfigView for the given file and hive path, creating a handler if needed. + ---@param filePath string|boolean Config file path, or false for an in-memory (orphan) view. + ---@param hivePath string|string[] The hive (nested key path) this view targets. + ---@param defaults? table Default values for the hive. + ---@param logger? Logger + ---@param noLoad? boolean Don't load the file immediately (default false). + ---@param schemaOpts? { schemaId: string, migrate: fun(config: table, current?: string, target: string): boolean } Schema id/migration for the backing handler (see ConfigHandler.get); applied only when the handler is first created for this file. + ---@return ConfigView? view + ---@return string? err + @get = (filePath, hivePath, defaults, logger, noLoad = false, schemaOpts) => + ConfigHandler or= require "l0.DependencyControl.ConfigHandler" + + if filePath + handler, msg = ConfigHandler\get filePath, logger, noLoad, schemaOpts + return nil, msg unless handler + return handler\getView hivePath, defaults + else + -- orphan view: in-memory only, no file backing (used for virtual modules) + handler = ConfigHandler nil, logger + return ConfigView handler, hivePath, defaults + + + ---Creates a view into a hive of the given ConfigHandler. + ---@param configHandler ConfigHandler|nil Backing handler, or nil for an in-memory (orphan) view. + ---@param hivePath string|string[] The hive (nested key path) this view targets. + ---@param defaults? table Default values for the hive. + new: (configHandler, hivePath, defaults) => + ConfigHandler or= require "l0.DependencyControl.ConfigHandler" + @__hivePath = "table" == type(hivePath) and hivePath or {hivePath} + @__configHandler = configHandler + + -- deprecated, provided for compatibility with DepCtrl < 0.7 + @section = @__hivePath + -- compat: expose file path directly on the view + @file = configHandler and configHandler.filePath + + if configHandler + success, msg = @refresh! + configHandler.logger\assert @userConfig, msgs.new.failedRetrieveHive, hivePath, msg + else + @userConfig = {} -- orphan view: no file backing + + setDefaults @, defaults + @config = setmetatable {}, { + __index: (_, k) -> + uc = @userConfig[k] + return @defaults[k] if uc == nil + def = @defaults[k] + -- a partially-populated user section still resolves its unset keys from the section default + return mergeSection uc, def if type(uc) == "table" and type(def) == "table" + return uc + __newindex: (_, k, v) -> + @userConfig[k] = v + __len: (tbl) -> return 0 + __ipairs: (tbl) -> error "numerically indexed config hive keys are not supported" + __pairs: (tbl) -> + merged = Common.copy @defaults + merged[k] = v for k, v in pairs @userConfig + return next, merged + } + @c = @config -- shortcut + + + -- Wraps each top-level default section in a copy-on-write proxy: reads fall through to the section's + -- defaults, and the first write into a section not yet present in the user config deep-copies that + -- section's defaults into it before applying the write. Only the top level is wrapped -- a section + -- already present in the user config is served per-key by mergeSection through @config. The proxy must + -- never be iterated here: its __pairs yields the underlying default keys, so descending into it would + -- fire the copy-on-write __newindex and materialize every default over the user's config on load. + setDefaults = (defaults) => + @defaults = defaults and Common.deepCopy(defaults) or {} + for section, contents in pairs @defaults + continue if type(contents) != "table" or type(section) == "string" and section\match "^__" + @defaults[section] = setmetatable {__targetMethodKey: section, __targetTable: contents}, { + __index: contents -- reads of unset keys fall through to the real defaults + __len: (proxy) -> #proxy.__targetTable + __newindex: (proxy, key, value) -> + -- first write into an absent section: copy its defaults into the user config, then write + @userConfig[proxy.__targetMethodKey] = Common.deepCopy proxy.__targetTable + @userConfig[proxy.__targetMethodKey][key] = value + __pairs: (proxy) -> next, proxy.__targetTable + __ipairs: (proxy) -> + i, n, orgTbl = 0, #proxy.__targetTable, proxy.__targetTable + -> + i += 1 + return i, orgTbl[i] if i <= n + } + + + ---Removes this view's hive from the config file. + ---@param waitLockTime? number Seconds to wait for the config lock. + ---@return boolean? success + ---@return string? err + delete: (waitLockTime) => + @userConfig, msg = @__configHandler\purgeHive @ + return nil, msg unless @userConfig + return @save waitLockTime + + + ---Copies values from a table or ConfigView into this view's user config. + ---@param tbl? table|ConfigView Source values. + ---@param keys? string[] Restrict the copy to these keys. + ---@param updateOnly? boolean Only overwrite keys already present in this view. + ---@param skipSameLengthTables? boolean Skip array values whose length matches the existing one. + ---@return boolean changesMade + import: (tbl, keys, updateOnly, skipSameLengthTables) => + tbl = tbl.userConfig if tbl.__class == @@ + changesMade = false + keySet = Common.makeSet keys if keys + + for k, v in pairs tbl + continue if keys and not keySet[k] or @userConfig[k] == v + continue if updateOnly and @config[k] == nil + isTable = type(v) == "table" + if isTable and skipSameLengthTables and type(@userConfig[k]) == "table" and #v == #@userConfig[k] + continue + continue if type(k) == "string" and k\sub(1,1) == "_" + @userConfig[k] = ConfigHandler\getSerializableCopy v + changesMade = true + + return changesMade + + + ---Returns whether this view's hive overlaps with another view on the same handler. + ---@param otherView ConfigView + ---@return boolean? overlapping nil when the views belong to different handlers. + ---@return string? err + isOverlappingView: (otherView) => + if @__configHandler != otherView.__configHandler + return nil, msgs.isOverlappingView.differentHandler\format otherView.__configHandler.filePath, + @__configHandler.filePath + + thisViewHivePathDepth, otherViewHivePathDepth = #@__hivePath, #otherView.__hivePath + + return true if thisViewHivePathDepth == 0 or otherViewHivePathDepth == 0 + + for i, key in ipairs @__hivePath + return false if key != otherView.__hivePath[i] + return true if i == thisViewHivePathDepth or i == otherViewHivePathDepth + + + ---Reloads only this view's hive from the config file. + ---@param waitLockTime? number Seconds to wait for the config lock. + ---@return boolean? success + ---@return string? err + load: (waitLockTime) => + return false unless @__configHandler and @__configHandler.filePath + @__configHandler\load @, waitLockTime + + + ---Refreshes this view's userConfig from the handler's in-memory config. + ---@return boolean? success + ---@return string? err + refresh: => + @userConfig, msg = @__configHandler\getHive @__hivePath + return if @userConfig + true + else nil, msg + + + ---Writes this view's hive to the config file. + ---@param waitLockTime? number Seconds to wait for the config lock. + ---@return boolean? success + ---@return string? err + save: (waitLockTime) => + return false unless @__configHandler and @__configHandler.filePath + @__configHandler\save @, waitLockTime + + + ---@deprecated Use `save`. Retained as an alias for callers written against DepCtrl < 0.7. + ---@param waitLockTime? number Seconds to wait for the config lock. + ---@return boolean? success + ---@return string? err + write: (waitLockTime) => @save waitLockTime + + ---Attaches this view to a different config file, without loading it (the caller loads separately). + ---@param filePath string Full path to the config file. + ---@return boolean? success + ---@return string? err + setFile: (filePath) => + ConfigHandler or= require "l0.DependencyControl.ConfigHandler" + logger = @__configHandler and @__configHandler.logger + handler, msg = ConfigHandler\get filePath, logger, true -- noLoad: caller loads separately + return nil, msg unless handler + @__configHandler = handler + @file = handler.filePath + return true + + ---Detaches this view from its config file, reverting to an in-memory (orphan) state. + ---@return boolean success + unsetFile: => + ConfigHandler or= require "l0.DependencyControl.ConfigHandler" + @__configHandler = ConfigHandler nil, @__configHandler and @__configHandler.logger + @file = nil + @userConfig = {} + return true + + ---Returns a ConfigView for a child hive of this view's config. + ---@param hivePath string|string[] Path to the child hive, relative to this view. + ---@param defaults? table Default values for the child view. + ---@param noLoad? boolean Skip loading the child view (the caller loads it separately). + ---@return ConfigView? view + ---@return string? err + getSectionHandler: (hivePath, defaults, noLoad) => + view, msg = @__configHandler\getView hivePath, defaults + return nil, msg unless view + view\load! unless noLoad + return view diff --git a/modules/l0/DependencyControl/Constants.moon b/modules/l0/DependencyControl/Constants.moon new file mode 100644 index 0000000..5a6ecb2 --- /dev/null +++ b/modules/l0/DependencyControl/Constants.moon @@ -0,0 +1,7 @@ +{ + DEPCTRL_NAME: "DependencyControl" + DEPCTRL_SHORT_NAME: "DepCtrl" + DEPCTRL_NAMESPACE: "l0.DependencyControl" + DEPCTRL_PRIVATE_GLOBAL_VAR_PREFIX: "__depCtrl" + DEPCTRL_FEED_URL: "https://raw.githubusercontent.com/TypesettingTools/DependencyControl/master/DependencyControl.json" +} diff --git a/modules/l0/DependencyControl/Crypto.moon b/modules/l0/DependencyControl/Crypto.moon new file mode 100644 index 0000000..4cb6ea1 --- /dev/null +++ b/modules/l0/DependencyControl/Crypto.moon @@ -0,0 +1,181 @@ +-- Cryptographic / hashing utilities. +-- Uses a fast native SHA-1 when one is available (CommonCrypto on macOS, libcrypto +-- on Linux, the Windows CryptoAPI), and falls back to a pure-Lua implementation +-- otherwise — so it always works, even headless / on platforms without the libs. + +ffi = require "ffi" +bit = require "bit" +band, bor, bxor, bnot = bit.band, bit.bor, bit.bxor, bit.bnot +lshift, rol, tobit, tohex = bit.lshift, bit.rol, bit.tobit, bit.tohex + +msgs = { + sha1: { + badPayload: "Expected a string payload to hash, got a '%s'." + } +} + +-- Formats a 20-byte digest buffer as a 40-character lowercase hex string. +digestToHex = (buf) -> table.concat ["%02x"\format buf[i] for i = 0, 19] + +-- Pure-Lua SHA-1 (reference / fallback). Assumes a string input. +sha1Lua = (msg) -> + h0, h1, h2, h3, h4 = 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0 + bytes = #msg + + -- append 0x80, pad with zeros until length ≡ 56 (mod 64) + msg ..= "\128" + while #msg % 64 != 56 + msg ..= "\0" + + -- append the original length in bits as a 64-bit big-endian integer + lenHi = math.floor bytes / 0x20000000 + lenLo = bytes * 8 % 0x100000000 + beBytes = (v) -> string.char( + band(math.floor(v / 0x1000000), 0xFF), band(math.floor(v / 0x10000), 0xFF), + band(math.floor(v / 0x100), 0xFF), band(v, 0xFF)) + msg ..= beBytes(lenHi) .. beBytes(lenLo) + + W = {} + for chunk = 1, #msg, 64 + for i = 0, 15 + b0, b1, b2, b3 = string.byte msg, chunk + i * 4, chunk + i * 4 + 3 + W[i] = bor lshift(b0, 24), lshift(b1, 16), lshift(b2, 8), b3 + for i = 16, 79 + W[i] = rol bxor(W[i - 3], W[i - 8], W[i - 14], W[i - 16]), 1 + + a, b, c, d, e = h0, h1, h2, h3, h4 + for i = 0, 79 + local f, k + if i < 20 + f, k = bor(band(b, c), band(bnot(b), d)), 0x5A827999 + elseif i < 40 + f, k = bxor(b, c, d), 0x6ED9EBA1 + elseif i < 60 + f, k = bor(band(b, c), bor(band(b, d), band(c, d))), 0x8F1BBCDC + else + f, k = bxor(b, c, d), 0xCA62C1D6 + temp = tobit rol(a, 5) + f + e + k + W[i] + e, d, c, b, a = d, c, rol(b, 30), a, temp + + h0 = tobit h0 + a + h1 = tobit h1 + b + h2 = tobit h2 + c + h3 = tobit h3 + d + h4 = tobit h4 + e + + tohex(h0) .. tohex(h1) .. tohex(h2) .. tohex(h3) .. tohex(h4) + +-- Attempts to set up a native SHA-1. Returns (fn, backendName) or nil. +-- Each fn takes a string and returns the 40-char hex digest. +setupNativeSha1 = -> + switch ffi.os + when "OSX" + -- CommonCrypto's CC_SHA1 is exported from libSystem (always loaded). + pcall ffi.cdef, "unsigned char* CC_SHA1(const void* data, uint32_t len, unsigned char* md);" + return unless pcall -> ffi.C.CC_SHA1 + digest = ffi.new "unsigned char[20]" + impl = (msg) -> + ffi.C.CC_SHA1 msg, #msg, digest + digestToHex digest + return impl, "CommonCrypto" + + when "Windows" + okLib, advapi = pcall ffi.load, "advapi32" + return unless okLib + pcall ffi.cdef, [[ + int CryptAcquireContextW(uintptr_t* phProv, const wchar_t* container, const wchar_t* provider, unsigned long provType, unsigned long flags); + int CryptCreateHash(uintptr_t hProv, unsigned int algId, uintptr_t hKey, unsigned long flags, uintptr_t* phHash); + int CryptHashData(uintptr_t hHash, const unsigned char* data, unsigned long len, unsigned long flags); + int CryptGetHashParam(uintptr_t hHash, unsigned long param, unsigned char* data, unsigned long* len, unsigned long flags); + int CryptDestroyHash(uintptr_t hHash); + ]] + PROV_RSA_FULL, CRYPT_VERIFYCONTEXT = 1, 0xF0000000 + CALG_SHA1, HP_HASHVAL = 0x8004, 2 + prov = ffi.new "uintptr_t[1]" + return if 0 == advapi.CryptAcquireContextW prov, nil, nil, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT + hProv = prov[0] + digest = ffi.new "unsigned char[20]" + dlen = ffi.new "unsigned long[1]" + impl = (msg) -> + hashPtr = ffi.new "uintptr_t[1]" + return sha1Lua msg if 0 == advapi.CryptCreateHash hProv, CALG_SHA1, 0, 0, hashPtr + hHash = hashPtr[0] + advapi.CryptHashData hHash, msg, #msg, 0 + dlen[0] = 20 + advapi.CryptGetHashParam hHash, HP_HASHVAL, digest, dlen, 0 + advapi.CryptDestroyHash hHash + digestToHex digest + return impl, "CryptoAPI" + + else + -- Linux and other Unix: OpenSSL libcrypto. + local libcrypto + for name in *{"libcrypto.so.3", "libcrypto.so.1.1", "libcrypto.so", "crypto"} + okLib, lib = pcall ffi.load, name + if okLib + libcrypto = lib + break + return unless libcrypto + digest = ffi.new "unsigned char[20]" + + -- Preferred: the non-deprecated EVP interface (OpenSSL 1.1+/3.0). + pcall ffi.cdef, [[ + const void* EVP_sha1(void); + void* EVP_MD_CTX_new(void); + void EVP_MD_CTX_free(void* ctx); + int EVP_DigestInit_ex(void* ctx, const void* type, void* engine); + int EVP_DigestUpdate(void* ctx, const void* data, size_t count); + int EVP_DigestFinal_ex(void* ctx, unsigned char* md, unsigned int* size); + ]] + if pcall -> libcrypto.EVP_MD_CTX_new + md = libcrypto.EVP_sha1! + impl = (msg) -> + ctx = libcrypto.EVP_MD_CTX_new! + return sha1Lua msg if ctx == nil + libcrypto.EVP_DigestInit_ex ctx, md, nil + libcrypto.EVP_DigestUpdate ctx, msg, #msg + libcrypto.EVP_DigestFinal_ex ctx, digest, nil + libcrypto.EVP_MD_CTX_free ctx + digestToHex digest + return impl, "OpenSSL (EVP)" + + -- Fallback for very old libcrypto: the legacy one-shot (deprecated in 3.0 + -- but still exported; FFI resolves it at runtime regardless). + pcall ffi.cdef, "unsigned char* SHA1(const unsigned char* d, size_t n, unsigned char* md);" + return unless pcall -> libcrypto.SHA1 + impl = (msg) -> + libcrypto.SHA1 msg, #msg, digest + digestToHex digest + return impl, "OpenSSL (SHA1)" + +-- Resolve the SHA-1 backend, but only trust a native one if it reproduces the +-- reference digest (guards against a mis-bound symbol or wrong digest length). +sha1Impl, sha1Backend = sha1Lua, "lua" +ok, native, backendName = pcall setupNativeSha1 +if ok and native + verified, digest = pcall native, "abc" + if verified and digest == sha1Lua "abc" + sha1Impl, sha1Backend = native, backendName + +---Cryptographic / hashing utilities backed by a native SHA-1 where available. +---@class Crypto +class Crypto + -- Name of the active SHA-1 backend ("CommonCrypto"/"OpenSSL"/"CryptoAPI"/"lua"). + @sha1Backend = sha1Backend + + ---Computes the SHA-1 digest of a string. + ---Accepts arbitrary binary data: Lua strings are byte-safe, so any byte sequence + ---(e.g. a file read in binary mode) hashes correctly. A raw FFI buffer must be + ---converted with ffi.string(buf, len) first. + ---Suitable for file integrity verification; not for security-sensitive use. + ---@param msg string The input bytes (may be binary). + ---@return string? digest A 40-character lowercase hex digest, or nil on invalid input. + ---@return string? err + @sha1 = (msg) -> + return nil, msgs.sha1.badPayload\format type(msg) unless type(msg) == "string" + sha1Impl msg + + -- The pure-Lua reference implementation, exposed for tests / explicit fallback. + @_sha1Lua = sha1Lua + +return Crypto diff --git a/modules/l0/DependencyControl/Downloader.moon b/modules/l0/DependencyControl/Downloader.moon new file mode 100644 index 0000000..0e80054 --- /dev/null +++ b/modules/l0/DependencyControl/Downloader.moon @@ -0,0 +1,637 @@ +-- Non-blocking download manager with SHA-1 verification. +-- Pure FFI implementation inspired by torque's DM.DownloadManager. +-- +-- macOS/Linux: libcurl multi interface — parallel, scheduled by libcurl +-- Windows: WinINet driver multiplexed by our round-robin scheduler (parallel) + +ffi = require "ffi" +lfs = require "lfs" +Enum = require "l0.DependencyControl.Enum" +FileOps = require "l0.DependencyControl.FileOps" +EventEmitter = require "l0.DependencyControl.EventEmitter" +Host = require "l0.DependencyControl.Host" + +msgs = { + addMissingArgs: "Required arguments #1 (url) and #2 (outfile) had the wrong type. Expected string, got '%s' and '%s'." + failedToOpen: "Could not open file '%s'." + noBackend: "No download backend available." + httpStatus: "Server returned HTTP status %d." + readFailed: "Connection error while reading response." + openUrlFailed: "Could not open URL '%s'." + curlInit: "Failed to initialize curl." + stalled: "Download stalled: no data received for %d seconds." + privateHostBlocked: "Refusing to download from a private, loopback, or link-local address as an SSRF safeguard (a feed or package could otherwise point DependencyControl at an internal host): %s. To allow this for local development/testing, set updates.blockPrivateHosts = false in the DependencyControl config." + nonHttpScheme: "Refusing to download from a non-http(s) URL as an SSRF safeguard (a feed or package could otherwise point DependencyControl at a local file or other scheme): %s." + tooManyRedirects: "Gave up following redirects while downloading '%s' (too many hops)." + redirectNoLocation: "Redirect response from '%s' had no Location header." +} + +-- Resolves a redirect Location (which may be relative) against the URL it came from. An absolute URL is +-- returned unchanged; "//host/path" inherits the base's scheme; "/path" keeps the base authority; anything +-- else is resolved against the base's directory. Used by the WinINet backend, which follows redirects by +-- hand (libcurl resolves them internally). +resolveRedirect = (base, location) -> + return location if location\match "^%a[%w+.%-]*://" + proto, authority, path = base\match "^(%a[%w+.%-]*)://([^/]*)(.*)$" + return location unless proto + return "#{proto}:#{location}" if location\sub(1, 2) == "//" + return "#{proto}://#{authority}#{location}" if location\sub(1, 1) == "/" + dir = path\match("^(.*/)") or "/" + "#{proto}://#{authority}#{dir}#{location}" + +-- Lifecycle state of a single download. +DownloadStatus = Enum "DownloadStatus", { + Queued: "queued" -- created, not yet started + Active: "active" -- transfer in progress + Finished: "finished" -- completed successfully + Failed: "failed" -- completed with an error + Cancelled: "cancelled" -- cancelled before completion +} + +-- statuses representing a download that is no longer in flight +isTerminalStatus = { + [DownloadStatus.Finished]: true + [DownloadStatus.Failed]: true + [DownloadStatus.Cancelled]: true +} + +-- Reports progress by emitting the downloader's Progress event, then returns +-- whether to keep going (a Progress listener may call cancel! to stop). +report = (manager, progress) -> + manager\_reportProgress progress + not manager.cancelled + +-- Backend-agnostic aggregate progress (0-100) from per-download state. +-- Relies on dl.bytesReceived / dl.totalBytes / dl.status, which every runner maintains. +computeProgress = (downloads) -> + total, now, allKnown, done = 0, 0, true, 0 + for dl in *downloads + if isTerminalStatus[dl.status] + done += 1 + total += dl.bytesReceived or 0 + now += dl.bytesReceived or 0 + else + if dl.totalBytes and dl.totalBytes > 0 + total += dl.totalBytes + now += dl.bytesReceived or 0 + else + allKnown = false + if total > 0 and allKnown + math.floor 100 * now / total + else + math.floor 100 * done / math.max #downloads, 1 + +-- Generic round-robin scheduler over a driver. This is the core scheduling logic +-- (the Windows production path, and the unit-tested path via a fake driver). +-- driver = { +-- start(dl) -> true | (false, errString) -- begin one transfer; set dl.totalBytes if known +-- step(dl) -> "more" | "done" | errString -- advance one chunk; update dl.bytesReceived +-- finish(dl) -> -- release one transfer's resources (idempotent) +-- shutdown() -> -- optional: release shared resources +-- } +multiplex = (manager, driver) -> + downloads = manager.downloads + queue = downloads + maxConnections = manager.maxConnections or 8 + stallTimeout = manager.stallTimeout + + active, queueIndex = {}, 1 + + -- Start the next queued download into an active slot. Returns the started download, or nil + -- when the queue is exhausted. A download that fails to start is finalized and skipped so + -- the slot stays filled. + startNext = -> + return nil if queueIndex > #queue + dl = queue[queueIndex] + queueIndex += 1 + dl.bytesReceived = 0 + ok, err = driver.start dl + unless ok + dl\_complete err or "failed to start download" + return startNext! + dl.status = DownloadStatus.Active + dl._lastProgressBytesReceived, dl._lastProgressAt = 0, os.time! + active[#active + 1] = dl + dl + + fillSlots = -> + while #active < maxConnections and queueIndex <= #queue and not manager.cancelled + break unless startNext! + + fillSlots! + + -- one pass per loop iteration steps every still-active transfer exactly once + while #active > 0 and not manager.cancelled + now = os.time! + remaining = {} + for dl in *active + if dl._cancelRequested + driver.finish dl + dl\_cancel! + else + status = driver.step dl + if status == "more" + dl\_notifyProgress! + if dl.bytesReceived > dl._lastProgressBytesReceived + -- progress made: reset the stall timer + dl._lastProgressBytesReceived, dl._lastProgressAt = dl.bytesReceived, now + remaining[#remaining + 1] = dl + elseif stallTimeout and stallTimeout > 0 and now - dl._lastProgressAt >= stallTimeout + driver.finish dl + dl\_complete msgs.stalled\format stallTimeout + else + -- no new bytes yet, but not stalled long enough to give up + remaining[#remaining + 1] = dl + elseif status == "done" + driver.finish dl + dl\_complete! + else + driver.finish dl + dl\_complete status + active = remaining + fillSlots! + -- report progress and allow cancellation between each round of steps + break unless report manager, computeProgress downloads + + -- cancel remaining individual downloads if the whole downloader is cancelled + for dl in *active + driver.finish dl + dl\_cancel! + for i = queueIndex, #queue + queue[i]\_cancel! + + driver.shutdown! if driver.shutdown + +-- Platform backend selection: sets defaultRunner(manager) and isInternetConnected(). +local defaultRunner, isInternetConnected + +if ffi.os != "Windows" + pcall ffi.cdef, "void* fopen(const char* path, const char* mode);" + pcall ffi.cdef, "int fclose(void* stream);" + pcall ffi.cdef, "int usleep(unsigned int usec);" + pcall ffi.cdef, [[ + void* curl_easy_init(void); + int curl_easy_setopt(void* handle, int option, ...); + void curl_easy_cleanup(void* handle); + int curl_easy_getinfo(void* handle, int info, ...); + const char* curl_easy_strerror(int errornum); + void* curl_multi_init(void); + int curl_multi_setopt(void* multi, int option, long value); + int curl_multi_add_handle(void* multi, void* easy); + int curl_multi_remove_handle(void* multi, void* easy); + int curl_multi_perform(void* multi, int* running); + int curl_multi_wait(void* multi, void* extra_fds, unsigned int extra_nfds, int timeout_ms, int* numfds); + void curl_multi_cleanup(void* multi); + typedef struct CURLMsg { + int msg; + void* easy_handle; + union { void* whatever; int result; } data; + } CURLMsg; + CURLMsg* curl_multi_info_read(void* multi, int* msgs_in_queue); + ]] + + curlNames = ffi.os == "OSX" and {"libcurl.4.dylib", "libcurl.dylib", "curl"} or + {"libcurl.so.4", "libcurl.so", "curl"} + local curl + for name in *curlNames + loaded, lib = pcall ffi.load, name + if loaded + curl = lib + break + + if curl + CURLOPT_WRITEDATA = 10001 -- write the response data to the file passed as a pointer + CURLOPT_URL = 10002 -- set the URL to fetch + CURLOPT_USERAGENT = 10018 -- set the User-Agent header + CURLOPT_FOLLOWLOCATION = 52 -- follow HTTP redirects + CURLOPT_MAXREDIRS = 68 -- cap how many redirects are followed + CURLOPT_PROTOCOLS = 181 -- bitmask of protocols the transfer may use + CURLOPT_REDIR_PROTOCOLS = 182 -- bitmask of protocols a redirect may switch to + CURLPROTO_HTTP = 1 + CURLPROTO_HTTPS = 2 + CURLOPT_PREREQFUNCTION = 20312 -- called before each connection (incl. every redirect hop); may abort it + CURL_PREREQFUNC_OK, CURL_PREREQFUNC_ABORT = 0, 1 + CURLOPT_FAILONERROR = 45 -- treat HTTP 4xx/5xx responses as errors + CURLOPT_NOPROGRESS = 43 -- disable curl's built-in progress meter + CURLOPT_CONNECTTIMEOUT = 78 -- abort if connecting takes longer than the specified number of seconds + CURLOPT_LOW_SPEED_LIMIT = 19 -- abort if the transfer speed is below this (in bytes/sec) for too long (see LOW_SPEED_TIME) + CURLOPT_LOW_SPEED_TIME = 20 -- the time (in seconds) the transfer speed should be below the limit before aborting + CURLINFO_SIZE_DOWNLOAD = 0x300008 -- total bytes downloaded so far + CURLINFO_CONTENT_LENGTH_DOWNLOAD = 0x30000F -- total expected size of the download, or -1 if unknown + CURLMSG_DONE = 1 -- a transfer completed (with either success or error) + CURLMOPT_MAX_TOTAL_CONNECTIONS = 13 -- max simultaneous connections of any kind + CURLMOPT_MAX_HOST_CONNECTIONS = 7 -- max simultaneous connections to the same host + + -- libcurl's varargs expect a C long for integer options; a bare Lua number + -- would be passed as a double, so cast explicitly. + setLong = (h, opt, v) -> curl.curl_easy_setopt h, opt, ffi.cast "long", v + -- cdata pointers can't be table keys reliably; key by address string instead. + key = (h) -> tostring ffi.cast "void *", h + + -- Fires before each connection (including every redirect hop), so a redirect to a private/internal + -- host is refused even though libcurl follows redirects itself. Requires curl >= 7.80, otherwise ignored. + pcall ffi.cdef, "typedef int (*dc_curl_prereq_cb)(void* clientp, char* conn_primary_ip, char* conn_local_ip, int conn_primary_port, int conn_local_port);" + prereqBlockPrivate = ffi.cast "dc_curl_prereq_cb", (_, primaryIp) -> + ok, private = pcall -> Host(ffi.string primaryIp)\isPrivate! + ok and private and CURL_PREREQFUNC_ABORT or CURL_PREREQFUNC_OK + + getDouble = (h, info) -> + out = ffi.new "double[1]" + curl.curl_easy_getinfo h, info, out + tonumber out[0] + + -- Unix uses curl's own multi scheduler rather than our round-robin loop. + defaultRunner = (manager) -> + downloads = manager.downloads + -- libcurl keeps excess transfers queued internally + multi = curl.curl_multi_init! + maxConnections = manager.maxConnections or 8 + curl.curl_multi_setopt multi, CURLMOPT_MAX_HOST_CONNECTIONS, ffi.cast "long", maxConnections + curl.curl_multi_setopt multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, ffi.cast "long", maxConnections + handleMap = {} + + for dl in *downloads + dl.bytesReceived = 0 + file = ffi.C.fopen dl.outfile, "wb" + if file == nil + dl\_complete msgs.failedToOpen\format dl.outfile + continue + handle = curl.curl_easy_init! + if handle == nil + ffi.C.fclose file + dl\_complete msgs.curlInit + continue + curl.curl_easy_setopt handle, CURLOPT_URL, dl.url + curl.curl_easy_setopt handle, CURLOPT_USERAGENT, "DependencyControl" + curl.curl_easy_setopt handle, CURLOPT_WRITEDATA, file + setLong handle, CURLOPT_FOLLOWLOCATION, 1 + setLong handle, CURLOPT_MAXREDIRS, 10 -- bound redirect fan-out + -- confine the transfer and any redirect to http(s), so a redirect can't reach file:// etc. + if manager.blockPrivateHosts + httpOnly = bit.bor CURLPROTO_HTTP, CURLPROTO_HTTPS + setLong handle, CURLOPT_PROTOCOLS, httpOnly + setLong handle, CURLOPT_REDIR_PROTOCOLS, httpOnly + curl.curl_easy_setopt handle, CURLOPT_PREREQFUNCTION, prereqBlockPrivate + setLong handle, CURLOPT_FAILONERROR, 1 + setLong handle, CURLOPT_NOPROGRESS, 1 + setLong handle, CURLOPT_CONNECTTIMEOUT, 30 + -- abort a transfer that drops below 1 byte/sec for stallTimeout seconds + if manager.stallTimeout and manager.stallTimeout > 0 + setLong handle, CURLOPT_LOW_SPEED_LIMIT, 1 + setLong handle, CURLOPT_LOW_SPEED_TIME, manager.stallTimeout + dl._handle, dl._file = handle, file + dl.status = DownloadStatus.Active + handleMap[key handle] = dl + curl.curl_multi_add_handle multi, handle + + drain = -> + pending = ffi.new "int[1]" + while true + multiStackInfo = curl.curl_multi_info_read multi, pending + break if multiStackInfo == nil + continue unless multiStackInfo.msg == CURLMSG_DONE + dl = handleMap[key multiStackInfo.easy_handle] + continue unless dl + res = multiStackInfo.data.result + dl.bytesReceived = getDouble dl._handle, CURLINFO_SIZE_DOWNLOAD + ffi.C.fclose dl._file + curl.curl_multi_remove_handle multi, dl._handle + curl.curl_easy_cleanup dl._handle + dl._file, dl._handle = nil + transportError = res != 0 and ffi.string(curl.curl_easy_strerror res) or nil + dl\_complete transportError -- fires finish callbacks (e.g. hash verification) + + -- releases an easy handle + its output file (idempotent) + releaseHandle = (dl) -> + ffi.C.fclose dl._file if dl._file + curl.curl_multi_remove_handle multi, dl._handle + curl.curl_easy_cleanup dl._handle + dl._file, dl._handle = nil + + running = ffi.new "int[1]" + running[0] = 1 + numfds = ffi.new "int[1]" + while running[0] > 0 + curl.curl_multi_perform multi, running + drain! + for dl in *downloads + continue unless dl._handle + if dl._cancelRequested + releaseHandle dl + dl\_cancel! + else + dl.bytesReceived = getDouble dl._handle, CURLINFO_SIZE_DOWNLOAD + contentLen = getDouble dl._handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD + dl.totalBytes = contentLen if contentLen > 0 + dl\_notifyProgress! + break unless report manager, computeProgress downloads + if running[0] > 0 + curl.curl_multi_wait multi, nil, 0, 100, numfds + ffi.C.usleep 10000 if numfds[0] == 0 + drain! + + -- finalize any survivors as cancelled (whole-downloader cancellation) + for dl in *downloads + if dl._handle + releaseHandle dl + dl\_cancel! + curl.curl_multi_cleanup multi + + else + defaultRunner = (manager) -> + dl\_complete msgs.noBackend for dl in *manager.downloads + + isInternetConnected = -> true -- best-effort: assume connected, let downloads report real errors + +else + ffiWin = require "l0.DependencyControl.helpers.ffi-windows" + + pcall ffi.cdef, [[ + void* InternetOpenW(const wchar_t* agent, unsigned long accessType, const wchar_t* proxy, const wchar_t* proxyBypass, unsigned long flags); + void* InternetOpenUrlW(void* session, const wchar_t* url, const wchar_t* headers, unsigned long headersLen, unsigned long flags, uintptr_t context); + int InternetReadFile(void* hFile, void* buffer, unsigned long toRead, unsigned long* read); + int InternetCloseHandle(void* h); + int InternetSetOptionW(void* hInternet, unsigned long option, void* buffer, unsigned long bufferLen); + int HttpQueryInfoW(void* hRequest, unsigned long infoLevel, void* buffer, unsigned long* bufferLen, unsigned long* index); + int HttpQueryInfoA(void* hRequest, unsigned long infoLevel, void* buffer, unsigned long* bufferLen, unsigned long* index); + int InternetGetConnectedState(unsigned long* flags, unsigned long reserved); + ]] + + haveKernel32 = ffiWin.haveKernel32 + haveWinInet, winInet = pcall ffi.load, "winInet" + + toWide = ffiWin.toWide + + INTERNET_FLAG_RELOAD = 0x80000000 -- force a reload from the server even if the content is cached + INTERNET_FLAG_NO_CACHE_WRITE = 0x04000000 -- don't commit this download to the cache + INTERNET_FLAG_NO_AUTO_REDIRECT = 0x00200000 -- don't auto-follow redirects; we validate each hop ourselves + INTERNET_OPTION_MAX_CONNS_PER_SERVER = 73 -- max simultaneous connections to the same HTTP/1.1 server + INTERNET_OPTION_MAX_CONNS_PER_1_0_SERVER = 74 -- max simultaneous connections to the same HTTP/1.0 server + HTTP_QUERY_STATUS_CODE = 19 -- HTTP response status code (e.g. 200) + HTTP_QUERY_CONTENT_LENGTH = 5 -- total expected size of the download, or -1 if unknown + HTTP_QUERY_LOCATION = 33 -- the Location response header (a redirect's target URL) + HTTP_QUERY_FLAG_NUMBER = 0x20000000 -- return the queried information as a number instead of a string (e.g. for status code or content length) + MAX_REDIRECTS = 10 -- cap on redirect hops we follow before giving up + CHUNK_SIZE = 16384 -- bytes to read for each running download per iteration of the scheduler loop (max WinINet buffer size) + + queryNumber = (request, info) -> + out = ffi.new "unsigned long[1]" + len = ffi.new "unsigned long[1]" + len[0] = 4 + ok = winInet.HttpQueryInfoW request, bit.bor(info, HTTP_QUERY_FLAG_NUMBER), out, len, nil + ok != 0 and tonumber(out[0]) or nil + + -- Reads a response header as a string (ANSI). A fixed buffer is plenty for a Location URL; the + -- private-host check tolerates a truncated value (it just fails to parse and refuses/loads elsewhere). + queryString = (request, info) -> + buf = ffi.new "char[2048]" + len = ffi.new "unsigned long[1]", 2048 + ok = winInet.HttpQueryInfoA request, info, buf, len, nil + ok != 0 and ffi.string(buf, len[0]) or nil + + if haveKernel32 and haveWinInet + -- A WinINet driver for `multiplex`: one request + output file per download, + -- advanced one chunk per step. The scheduler round-robins across them. + makeWinINetDriver = (manager, maxConnectionsPerServer = 8) -> + do + -- Lift the Windows-default 2-connections-per-server cap so all queued transfers can run at once; + -- otherwise a 3rd concurrent InternetOpenUrlW to the same host blocks and times out. + optVal = ffi.new "unsigned long[1]", maxConnectionsPerServer + winInet.InternetSetOptionW nil, INTERNET_OPTION_MAX_CONNS_PER_SERVER, optVal, 4 + winInet.InternetSetOptionW nil, INTERNET_OPTION_MAX_CONNS_PER_1_0_SERVER, optVal, 4 + session = winInet.InternetOpenW toWide("DependencyControl"), 0, nil, nil, 0 + buffer = ffi.new "char[?]", CHUNK_SIZE + read = ffi.new "unsigned long[1]" + { + start: (dl) -> + outFileHandle, err = io.open dl.outfile, "wb" + return false, (err or msgs.failedToOpen\format dl.outfile) unless outFileHandle + + -- Follow redirects by hand (NO_AUTO_REDIRECT) so we can validate each hop's host: WinINet + -- would otherwise re-resolve and connect to a redirect target with no hook to inspect it. + fail = (msg) -> + outFileHandle\close! + false, msg + url, redirectsLeft, request = dl.url, MAX_REDIRECTS, nil + while true + if manager.blockPrivateHosts + return fail msgs.nonHttpScheme\format url unless url\match "^https?://" + host = Host.fromUrl url + return fail msgs.privateHostBlocked\format url if host and host\isPrivate! + + request = winInet.InternetOpenUrlW session, toWide(url), nil, 0, + bit.bor(INTERNET_FLAG_RELOAD, INTERNET_FLAG_NO_CACHE_WRITE, INTERNET_FLAG_NO_AUTO_REDIRECT), 0 + return fail msgs.openUrlFailed\format url if request == nil + + status = queryNumber request, HTTP_QUERY_STATUS_CODE + if status and status >= 300 and status < 400 + location = queryString request, HTTP_QUERY_LOCATION + winInet.InternetCloseHandle request + request = nil + return fail msgs.tooManyRedirects\format dl.url if redirectsLeft <= 0 + return fail msgs.redirectNoLocation\format url unless location + url, redirectsLeft = resolveRedirect(url, location), redirectsLeft - 1 + elseif status and status >= 400 + winInet.InternetCloseHandle request + return fail msgs.httpStatus\format status + else + break + + dl._request, dl._outFileHandle = request, outFileHandle + dl.totalBytes = queryNumber request, HTTP_QUERY_CONTENT_LENGTH + true + + step: (dl) -> + return msgs.readFailed if 0 == winInet.InternetReadFile dl._request, buffer, CHUNK_SIZE, read + n = tonumber read[0] + return "done" if n == 0 + dl._outFileHandle\write ffi.string buffer, n + dl.bytesReceived += n + "more" + + finish: (dl) -> + winInet.InternetCloseHandle dl._request if dl._request + dl._outFileHandle\close! if dl._outFileHandle + dl._request, dl._outFileHandle = nil + + shutdown: -> + winInet.InternetCloseHandle session + } + + defaultRunner = (manager) -> + multiplex manager, makeWinINetDriver manager, manager.maxConnections + + else + defaultRunner = (manager) -> + dl\_complete msgs.noBackend for dl in *manager.downloads + + isInternetConnected = -> + return true unless haveWinInet + flags = ffi.new "unsigned long[1]" + winInet.InternetGetConnectedState(flags, 0) != 0 + +---A single download: its URL, output path, transfer state, and event callbacks. +---Events (see Download.Event): Progress (data arrived), Finish (reached a terminal +---status). A Finish listener may downgrade the status via markFailed (e.g. for a +---failed hash verification). The current state is exposed via @status (Download.Status). +---@class Download: EventEmitter +class Download extends EventEmitter + @Status = DownloadStatus + @Event = Enum "DownloadEvent", { Progress: "progress", Finish: "finish" } + + ---Creates a single download in the Queued state. + ---@param url string + ---@param outfile string Full output path. + ---@param id? number An identifier assigned by the Downloader. + new: (@url, @outfile, @id) => + super! + @bytesReceived = 0 + @totalBytes = nil + @status = DownloadStatus.Queued + @error = nil + + ---Requests cancellation of this download. The downloader releases its + ---resources and sets the status to Cancelled on its next scheduling pass. + cancel: => @_cancelRequested = true + + ---Marks the download as failed (e.g. from a Finish listener performing + ---hash verification). + ---@param err string The failure reason. + markFailed: (err) => + @error = err + @status = @@Status.Failed + + ---Part of the download-runner callback contract: a runner calls this to fire this download's + ---Progress listeners as bytes arrive. + _notifyProgress: => @_emit @@Event.Progress + + ---Part of the download-runner callback contract: a runner calls this when the transfer ends to + ---finalize it (success, or a transport-level error) and fire Finish listeners (which may downgrade + ---the status via `markFailed`). Idempotent — only the first call takes effect. + ---@param transportError? string A transport-level error, if any. + _complete: (transportError) => + return if @_finalized + @_finalized = true + if transportError + @error = transportError + @status = @@Status.Failed + else + @status = @@Status.Finished + @_emit @@Event.Finish + + ---Part of the download-runner callback contract: a runner calls this to finalize the download as + ---cancelled and fire Finish listeners. Idempotent. + _cancel: => + return if @_finalized + @_finalized = true + @status = @@Status.Cancelled + @_emit @@Event.Finish + + +---Options accepted by the Downloader constructor. +---@class DownloaderOptions +---@field stallTimeout? number Seconds a transfer may receive no data before it's aborted; 0/false disables stall detection. +---@field maxConnections? integer Maximum simultaneous transfers (also the per-server connection limit); excess transfers queue. +---@field blockPrivateHosts? boolean Refuse URLs whose host is a private/loopback/link-local address (an SSRF guard). + +---Manages a set of concurrent downloads. This is DepCtrl's own engine; the +---DM.DownloadManager-compatible API lives in l0.DependencyControl.DownloadManager. +---Events (see Downloader.Event): Progress (overall %), Finished (await completed). +---@class Downloader: EventEmitter +class Downloader extends EventEmitter + @Download = Download + @Event = Enum "DownloaderEvent", { Progress: "progress", Finished: "finished" } + -- Exposed so tests (and custom runners) can drive the round-robin scheduler + -- with an injected driver. + @multiplex = multiplex + + -- Maximum simultaneous transfers (also applied as the per-server connection limit on each + -- backend). Excess downloads are queued and started as slots free. + maxConnections: 8 + + -- The number of seconds a transfer can go without receiving any data before we consider + -- it stalled and abort it. Set to 0 or false to disable stall detection. + stallTimeout: 30 + + ---Creates a downloader. + ---@param runner? fun(downloader: Downloader) Overrides the transfer implementation (defaults to the platform backend). + ---@param options? DownloaderOptions Optional downloader settings. + new: (runner, options = {}) => + super! + + @stallTimeout = options.stallTimeout if options.stallTimeout != nil + @maxConnections = options.maxConnections if options.maxConnections != nil + @blockPrivateHosts = options.blockPrivateHosts if options.blockPrivateHosts != nil + + @downloads = {} + @cancelled = false + @_runner = runner or defaultRunner + + ---Queues a download. Transfers happen later, in await. + ---Register progress/finish listeners on the returned Download as needed. + ---@param url string + ---@param outfile string Full output path (relative paths unsupported). + ---@param sha1? string Expected SHA-1 hash; verified automatically on finish. + ---@return Download? download + ---@return string? err + addDownload: (url, outfile, sha1) => + unless type(url) == "string" and type(outfile) == "string" + return nil, msgs.addMissingArgs\format type(url), type(outfile) + + -- SSRF guard (opt-in per instance): confine fetches to http(s) and reject private/internal hosts. + -- The scheme check also covers URLs with no host (file://, ...) that the private-IP check can't see. + if @blockPrivateHosts + return nil, msgs.nonHttpScheme\format url unless url\match "^https?://" + host = Host.fromUrl url + return nil, msgs.privateHostBlocked\format url if host and host\isPrivate! + + FileOps.mkdir outfile, true, true + + @_lastId = (@_lastId or 0) + 1 + download = Download url, outfile, @_lastId + + if type(sha1) == "string" + expected = sha1\lower! + -- piggyback on the finish event to verify the downloaded file's hash + download\on Download.Event.Finish, (dl) -> + return unless dl.status == Download.Status.Finished -- only verify successful transfers + ok, msg = FileOps.verifyHash dl.outfile, expected, FileOps.HashType.SHA1 + dl\markFailed msg unless ok + + @downloads[#@downloads + 1] = download + download + + ---Performs all queued downloads, blocking until they finish or are cancelled. + ---Subscribe to Progress/Finished via on; a Progress listener may call cancel!. + ---Inspect each download's final state via its @status (Download.Status). + ---@param onProgress? fun(downloader: Downloader, percent: number) Called with this downloader and the aggregate progress (0-100) on each Progress event, for the duration of this call only. + ---@return Downloader self for chaining + await: (onProgress) => + @on @@Event.Progress, onProgress if onProgress + @_runner @ + @off @@Event.Progress, onProgress if onProgress + @_emit @@Event.Finished + return @ + + ---@return number progress Current aggregate progress (0-100). + progress: => computeProgress @downloads + + ---Part of the download-runner callback contract: a runner calls this to report the manager's + ---aggregate progress and fire the Downloader's Progress listeners. + ---@param percent number Aggregate progress, 0-100. + _reportProgress: (percent) => @_emit @@Event.Progress, percent + + ---Cancels all remaining downloads (e.g. from within a Progress listener). + cancel: => @cancelled = true + + ---Removes all downloads and resets state. + ---Empties the array in place so external references stay valid. + clear: => + @downloads[i] = nil for i = #@downloads, 1, -1 + @cancelled = false + + ---@return boolean connected Whether an internet connection appears to be available. + isInternetConnected: => isInternetConnected! + +UnitTestSuite = require "l0.DependencyControl.UnitTestSuite" +return UnitTestSuite\withTestExports Downloader, {:resolveRedirect} diff --git a/modules/l0/DependencyControl/Enum.moon b/modules/l0/DependencyControl/Enum.moon new file mode 100644 index 0000000..f87aed0 --- /dev/null +++ b/modules/l0/DependencyControl/Enum.moon @@ -0,0 +1,122 @@ +Logger = require "l0.DependencyControl.Logger" + +reservedKeys = { + "describe", + "elements" + "keys", + "name", + "test", + "values" +} + +reservedKeySet = {v, true for v in *reservedKeys} + +msgs = { + __index: { + invalidKeyAccess: "Cannot access invalid key '%s' on Enum '%s'" + } + __newindex: { + immutableError: "Cannot assign field '%s' to '%s' on immutable Enum '%s'." + } + new: { + valueAlreadyTaken: "Could not define '%s' in enum '%s': value %s is already taken by '%s'." + keyAlreadyDefined: "Cannot redefine key '%s' in enum '%s'." + noReservedKeys: "Key may not be any of the reserved words [#{table.concat reservedKeys, ', '}] or start with '__' (was '%s')." + missingOrInvalidName: "Missing or invalid Enum name (expected a string, got a '%s')." + } + describe: { + valueNotDefined: "Value '%s' is not defined in enum '%s'." + } + validate: { + argPrefix: "Argument %s: " + invalidValue: "%sInvalid value '%s' for enum '%s'." + } +} + +---An immutable enumeration type with value/key reverse lookup. +---@class Enum +class Enum + @logger = Logger fileBaseName: "DependencyControl.Enum" + ---Reports whether `k` is reserved as an enum key — a built-in member name or `__`-prefixed. + ---@param k string + ---@return boolean reserved + @isReservedKey = (k) => + return type(k) == "string" and (k\sub(1,2) == "__" or reservedKeySet[k]) or false + + + ---Creates an enum from a table of key/value pairs or a list of names. + ---@param name string + ---@param values table Key/value pairs, or a list of names whose value defaults to their position. + ---@param __logger? Logger + new: (@name, values, @__logger = @@logger) => + @__logger\assert type(@name) == "string", msgs.new.missingOrInvalidName, Logger\describeType @name + @elements, @__keysByValue, @values, @keys = {}, {}, {}, {} + + for k, v in pairs values + -- we support lists as input, but we do not support numerical keys, which is sane + if "number" == type k + k, v = v, k + + @__logger\assert not @@isReservedKey(k), msgs.new.noReservedKeys, k + @__logger\assert @elements[k] == nil, msgs.new.keyAlreadyDefined, k, @name + @__logger\assert @__keysByValue[v] == nil, msgs.new.valueAlreadyTaken, k, @name, v, @__keysByValue[v] + + @elements[k], @__keysByValue[v] = v, k + table.insert @values, v + table.insert @keys, k + + meta = getmetatable @ + clsIdx = meta.__index + + setmetatable @, setmetatable { + __index: (k) => + if @elements[k] != nil + return @elements[k] + + v = switch type clsIdx + when "function" then clsIdx @, k + when "table" then clsIdx[k] + return v if v != nil + + @__logger\error msgs.__index.invalidKeyAccess, k, @name + + __newindex: (k, v) => + @__logger\error msgs.__newindex.immutableError, k, v, @name + }, clsIdx + + + ---Returns whether the given key is defined in this enum. + ---@param key string + ---@return boolean defined + ---@return any value The value mapped to the key, or nil if undefined. + test: (key) => + val = @elements[key] + return val != nil and true or false, val + + + ---Returns a human-readable description of the given value(s) in this enum. + ---@param values? any A single value, or a list of values to look up. If not provided, returns all keys. + ---@param join? string|boolean Separator string for joining multiple keys, true for ", ", or false to return a list (default true). + ---@param pattern? fun(key: string, value: any): string A function to format the key/value pair for display (default " ()"). + ---@return string|string[] A single string when a single value is provided, or a list of strings when multiple values are provided (or join is false). + describe: (values = @values, pattern = ((key, value) -> "#{value} (#{key})"), join = true) => + values = {values} if "table" != type values + + keys = for v in *values + key = @__keysByValue[v] + @__logger\assert key != nil, msgs.describe.valueNotDefined, v, @name + pattern key, v + + return join and table.concat(keys, join == true and ', ' or join) or keys + + ---Validates that a value is a member of this enum. + ---@param value any + ---@param argName? string Argument name to include in the error message. + ---@return boolean? valid True when the value is a member, nil otherwise. + ---@return string? err Validation error message when invalid. + validate: (value, argName) => + if value == nil or @__keysByValue[value] == nil + prefix = argName != nil and msgs.validate.argPrefix\format(argName) or "" + return nil, msgs.validate.invalidValue\format prefix, value, @name + + return true diff --git a/modules/l0/DependencyControl/EventEmitter.moon b/modules/l0/DependencyControl/EventEmitter.moon new file mode 100644 index 0000000..71a0969 --- /dev/null +++ b/modules/l0/DependencyControl/EventEmitter.moon @@ -0,0 +1,38 @@ +---Minimal event registration mixin: on(event, cb) / off(event, cb) / _emit(event, ...). +---Subclasses provide an `@Event` Enum that defines the valid event values. +---@class EventEmitter +class EventEmitter + new: => + @_listeners = {} + + ---Registers a callback for an event. + ---@param event any The event value (a member of the subclass's `@Event` enum). + ---@param callback fun(self: EventEmitter, ...) Called with the emitter instance plus any event arguments. + ---@return EventEmitter self for chaining + on: (event, callback) => + valid, err = @@Event\validate event, "event" + error err unless valid + listeners = @_listeners[event] + unless listeners + listeners = {} + @_listeners[event] = listeners + listeners[#listeners + 1] = callback + return @ + + ---Unregisters a previously-registered callback for an event. + ---@param event any The event value. + ---@param callback function The exact callback previously passed to on(). + ---@return EventEmitter self for chaining + off: (event, callback) => + listeners = @_listeners[event] + return @ unless listeners + for i = #listeners, 1, -1 + table.remove listeners, i if listeners[i] == callback + return @ + + -- Invokes all listeners for an event with (self, ...). Iterates a snapshot so + -- a listener may safely on/off during dispatch. + _emit: (event, ...) => + listeners = @_listeners[event] + return unless listeners + cb @, ... for cb in *[l for l in *listeners] diff --git a/modules/l0/DependencyControl/FeedInventory.moon b/modules/l0/DependencyControl/FeedInventory.moon new file mode 100644 index 0000000..e437e4c --- /dev/null +++ b/modules/l0/DependencyControl/FeedInventory.moon @@ -0,0 +1,322 @@ +constants = require "l0.DependencyControl.Constants" +Common = require "l0.DependencyControl.Common" +Enum = require "l0.DependencyControl.Enum" + +---How a feed was discovered — the source it came from. A feed can have several. +---@alias FeedProvenance +---| "official-depctrl" # OfficialDepCtrl: DependencyControl's own feed +---| "official-known" # OfficialKnown: a feed advertised in DependencyControl's own feed's knownFeeds +---| "user-extra" # UserExtra: a feed in the user's extraFeeds (a discovery root) +---| "package-declared" # PackageDeclared: the feed an installed package declares for its own updates +---| "package-override" # PackageOverride: an installed package's per-package userFeed override +---| "dependency-advertised" # DependencyAdvertised: a feed advertised in an installed package's requiredModules +---| "transitive-known" # TransitiveKnown: a feed advertised in another fetched feed's knownFeeds (crawl only) +Provenance = Enum "FeedProvenance", { + OfficialDepCtrl: "official-depctrl" + OfficialKnown: "official-known" + UserExtra: "user-extra" + PackageDeclared: "package-declared" + PackageOverride: "package-override" + DependencyAdvertised: "dependency-advertised" + TransitiveKnown: "transitive-known" +} + +-- Display order for a feed's provenance, most authoritative origin first. Pinned here because Enum.values +-- follows hash order, not definition order. +provenanceOrder = { + Provenance.OfficialDepCtrl + Provenance.OfficialKnown + Provenance.UserExtra + Provenance.PackageDeclared + Provenance.PackageOverride + Provenance.DependencyAdvertised + Provenance.TransitiveKnown +} + +---Which crawl budget a truncation hit. +---@alias FeedCrawlLimit +---| "per-feed" # PerFeed: the cap on how many untrusted feeds one feed may contribute +---| "per-root" # PerRoot: the per-subtree budget for untrusted expansion +---| "depth" # Depth: the crawl-depth limit; a feed at this depth is left unfetched +CrawlLimit = Enum "FeedCrawlLimit", { + PerFeed: "per-feed" + PerRoot: "per-root" + Depth: "depth" +} + +-- Ensures inventoryEntriesByUrl has a (possibly blank) entry for url; returns it, or nil for an invalid url. +ensureInventoryEntry = (inventoryEntriesByUrl, url) -> + return nil unless type(url) == "string" and #url > 0 + entry = inventoryEntriesByUrl[url] + unless entry + entry = {:url, provenance: {}, packages: {}, advertisedBy: {}} + inventoryEntriesByUrl[url] = entry + entry + +-- Ensures inventoryEntriesByUrl has an entry for url and tags it with the given provenance source; returns the entry (nil for +-- an invalid url). The `packages`/`advertisedBy` sets are filled by the caller. +addSource = (inventoryEntriesByUrl, url, source) -> + entry = ensureInventoryEntry inventoryEntriesByUrl, url + entry.provenance[source] = true if entry + entry + +-- The sorted keys of a set, as a list. +getSortedKeys = (set) -> + keys = [k for k in pairs set] + table.sort keys + keys + +-- A shallow copy of `list` with `item` appended, leaving the original untouched (for per-node crawl routes). +appended = (list, item) -> + copy = [x for x in *list] + copy[#copy + 1] = item + copy + +-- Cap on how many dropped feed URLs one truncation event records; its `dropped` count stays exact. +maxDropSample = 50 + +-- The feed a package's persisted `currentSource` resolves to (nil when absent), via UpdateTask's shared resolver. +resolveCurrentSource = (pkg, modulesSection) -> + src = pkg.currentSource + return nil unless type(src) == "table" + + UpdateTask or= require "l0.DependencyControl.UpdateTask" + UpdateTask.resolveSourceUrl src, pkg.feed, pkg.userFeed, modulesSection + +---A reachable feed with the sources it was discovered through and its trust status. +---@class FeedInventoryEntry +---@field url string The feed URL. +---@field provenance FeedProvenance[] The sources this feed was found through, in a stable order (empty for a feed present only in `trustedFeeds`). +---@field packages string[] Sorted namespaces of installed packages that declare, override to, or advertise this feed (empty when none). +---@field advertisedBy string[] Sorted URLs of fetched feeds that list this feed in their knownFeeds (crawl only). +---@field trustStatus FeedTrustStatus The feed's trust status. +---@field blockedBy? BlockedFeedEntry The block entry matching this feed when `trustStatus` is blocked (carries the reason and official/user flag). +---@field fetched? boolean True when `crawl` successfully read this feed; nil for `gather` (offline) or a feed the crawl couldn't reach. +---@field lastFetchedAt? integer Unix time this feed was last successfully fetched into the persistent cache; nil if it was never cached. +---@field inUse? boolean True when this feed is the effective update source (override else declared feed) of an installed package. +---@field inTrustedFeeds? boolean True when the feed is in the user's `trustedFeeds` — a trust-only listing that grants trust without acting as a discovery source. + +---The crawl budgets, keyed by `FeedCrawlLimit`; a missing key falls back to a built-in default. +---@alias FeedCrawlLimits table + +---A budget the crawl hit, with enough context to act on it (block a feed, file a report). +---@class FeedCrawlTruncation +---@field limit FeedCrawlLimit Which budget was hit. +---@field limitValue integer The configured value of that budget. +---@field feed string The feed being processed when the limit hit (the advertiser for per-feed/per-root, the unfetched feed itself for depth). +---@field root string The config-derived feed whose subtree this occurred in. +---@field depth integer The crawl depth of `feed` (0 for a config-derived root). +---@field route string[] The feed URLs from `root` to `feed` inclusive — how the crawl reached it. +---@field dropped integer How many advertised feeds this event left unexplored (0 for depth, whose single unfetched feed is `feed`). +---@field droppedUrls string[] The dropped feed URLs, capped to a sample when `dropped` exceeds it; empty for depth. + +---What a crawl explored and where it stopped short. +---@class FeedCrawlStats +---@field fetched integer How many feeds had their `knownFeeds` read. +---@field truncated boolean Whether any budget was hit (a quick "results incomplete?" check). +---@field truncations FeedCrawlTruncation[] Each budget hit, in crawl order, for surfacing or acting on. + +---Enumerates the feeds DependencyControl knows about — from the user config, installed packages, and its +---official trust lists — each tagged with the sources it was found through and its trust status, for the Manage +---Feeds UI. `gather` is network-free; `crawl` additionally discovers transitively-advertised feeds through an +---injected feed loader, bounding untrusted expansion. +---@class FeedInventory +class FeedInventory + @Provenance = Provenance + @CrawlLimit = CrawlLimit + + ---The built-in crawl budgets, used for any budget the config doesn't set (and the config's own default). + ---@type FeedCrawlLimits + @defaultCrawlLimits = { + [CrawlLimit.Depth]: 7 + [CrawlLimit.PerRoot]: 50 + [CrawlLimit.PerFeed]: 25 + } + + ---@param config ConfigView The global config view; its `c` holds `extraFeeds`/`trustedFeeds`/`macros`/`modules`, `fetchUntrustedFeeds`, and `feedCrawlLimits`. + ---@param feedTrust FeedTrust The trust model, for the official feeds and trust/block queries. + ---@param feedLoader FeedLoader Loads feeds during a crawl and holds the feed cache read for last-fetch times. + new: (@config, @feedTrust, @feedLoader) => + + ---The feed a package effectively updates from: its remembered `currentSource` (resolved), falling back to + ---its override (`userFeed`) or declared `feed`. + ---@param pkg table An installed package's config entry. + ---@param modulesSection table The modules config section, for resolving a provider `currentSource`. + ---@return string? url The feed the package updates from, or nil when it declares none. + @getEffectiveSource = (pkg, modulesSection) -> + resolveCurrentSource(pkg, modulesSection) or pkg.userFeed or pkg.feed + + ---Collects the feeds reachable from config, installed packages, and the official trust lists into a + ---`url -> raw entry` map (provenance/packages/advertisedBy still as sets). Network-free. + ---@private + ---@return table inventoryEntriesByUrl + __collectConfigFeeds: => + inventoryEntriesByUrl = {} + tagPackage = (url, source, namespace) -> + entry = addSource inventoryEntriesByUrl, url, source + entry.packages[namespace] = true if entry + + for url in pairs @feedTrust\getOfficialTrustedFeeds! + addSource inventoryEntriesByUrl, url, url == constants.DEPCTRL_FEED_URL and Provenance.OfficialDepCtrl or Provenance.OfficialKnown + + c = @config.c + addSource inventoryEntriesByUrl, url, Provenance.UserExtra for url in *(c.feeds.extraFeeds or {}) + -- trustedFeeds grant trust but aren't a discovery source, so they get no provenance — just the flag + for url in *(c.feeds.trustedFeeds or {}) + entry = ensureInventoryEntry inventoryEntriesByUrl, url + entry.inTrustedFeeds = true if entry + + modulesSection = c[Common.ScriptTypeSection[Common.ScriptType.Module]] or {} + for scriptType in *Common.ScriptType.values + for namespace, pkg in pairs (c[Common.ScriptTypeSection[scriptType]] or {}) + continue unless type(pkg) == "table" + tagPackage pkg.feed, Provenance.PackageDeclared, namespace + tagPackage pkg.userFeed, Provenance.PackageOverride, namespace + for dep in *(pkg.requiredModules or {}) + tagPackage dep.feed, Provenance.DependencyAdvertised, namespace if type(dep) == "table" + effective = FeedInventory.getEffectiveSource pkg, modulesSection + inventoryEntriesByUrl[effective].inUse = true if type(effective) == "string" and inventoryEntriesByUrl[effective] + + return inventoryEntriesByUrl + + ---Collapses each raw entry's provenance/package/advertisedBy sets to stable-ordered lists, attaches the + ---trust status (and, for a blocked feed, the block entry that matches it), and stamps the feed's last + ---fetch time from the persistent cache. + ---@private + ---@param inventoryEntriesByUrl table The raw entries to finalize. + ---@return FeedInventoryEntry[] feeds + __finalize: (inventoryEntriesByUrl) => + cache = @feedLoader.cache + feeds = {} + for url, entry in pairs inventoryEntriesByUrl + entry.provenance = [p for p in *provenanceOrder when entry.provenance[p]] + entry.packages = getSortedKeys entry.packages + entry.advertisedBy = getSortedKeys entry.advertisedBy + entry.trustStatus, entry.blockedBy = @feedTrust\getTrustStatus url + meta = cache\getMeta url + entry.lastFetchedAt = meta.cachedAt if meta and meta.cachedAt + feeds[#feeds + 1] = entry + feeds + + ---Gathers the known feeds from config, installed packages, and the official trust lists. Fetches nothing. + ---@return FeedInventoryEntry[] feeds + gather: => @__finalize @__collectConfigFeeds! + + ---Returns the namespaces of installed packages that effectively update from the given feed URL. + ---@param feedUrl string The feed URL to check. + ---@return string[] namespaces Sorted namespaces whose effective source is that feed. + getPackagesSourcedFrom: (feedUrl) => + c = @config.c + modulesSection = c[Common.ScriptTypeSection[Common.ScriptType.Module]] or {} + matched = {} + for scriptType in *Common.ScriptType.values + for namespace, pkg in pairs (c[Common.ScriptTypeSection[scriptType]] or {}) + continue unless type(pkg) == "table" + matched[#matched + 1] = namespace if FeedInventory.getEffectiveSource(pkg, modulesSection) == feedUrl + table.sort matched + matched + + ---Loads a feed's `knownFeeds` URLs through the shared feed loader, or nil when the feed can't be loaded. + ---@private + ---@param url string The feed URL to read. + ---@return string[]? knownFeeds + __loadKnownFeeds: (url) => + ok, feed = pcall @feedLoader.load, @feedLoader, url + ok and feed and feed.data and feed\getKnownFeeds! or nil + + ---Fetches feeds and discovers transitively-advertised ones by crawling the `knownFeeds` graph out from the + ---config-derived feeds; untrusted expansion is bounded, so check `stats.truncated` for incomplete results. + ---@return FeedInventoryEntry[] feeds The known feeds, enriched with what the crawl discovered. + ---@return FeedCrawlStats stats What the crawl explored and where it stopped short. + crawl: => + inventoryEntriesByUrl = @__collectConfigFeeds! + stats = @__crawlKnownFeeds inventoryEntriesByUrl + return @__finalize(inventoryEntriesByUrl), stats + + ---Breadth-first crawl of the `knownFeeds` graph, extending inventoryEntriesByUrl in place with the transitively-discovered + ---feeds under the untrusted-expansion bounds. Each config-derived feed is its own budget subtree, so a + ---malicious subtree can't starve the others. + ---@private + ---@param inventoryEntriesByUrl table The config-derived feeds to start from; extended in place. + ---@return FeedCrawlStats stats + __crawlKnownFeeds: (inventoryEntriesByUrl) => + c = @config.c + limits = c.feeds.crawlLimits or {} + defaults = @@defaultCrawlLimits + maxDepth = limits[CrawlLimit.Depth] or defaults[CrawlLimit.Depth] + maxPerRoot = limits[CrawlLimit.PerRoot] or defaults[CrawlLimit.PerRoot] + maxPerFeed = limits[CrawlLimit.PerFeed] or defaults[CrawlLimit.PerFeed] + stats = {fetched: 0, truncated: false, truncations: {}} + + record = (limit, feedUrl, root, depth, route, limitValue, drops) -> + stats.truncated = true + stats.truncations[#stats.truncations + 1] = { + :limit, feed: feedUrl, :root, :depth, :route, :limitValue + dropped: drops and drops.count or 0 + droppedUrls: drops and drops.sample or {} + } + + -- accumulate dropped feed URLs into a bounded sample while keeping an exact count + newDrops = -> {count: 0, sample: {}} + drop = (drops, url) -> + drops.count += 1 + drops.sample[#drops.sample + 1] = url if #drops.sample < maxDropSample + + -- BFS frontier seeded with the config-derived feeds; each is its own subtree root for budgeting + queue, visited, untrustedPerRoot = {}, {}, {} + for url, entry in pairs inventoryEntriesByUrl + -- don't crawl feeds with no discovery provenance (e.g. orphaned `trustedFeeds` entries) + continue unless next entry.provenance + queue[#queue + 1] = {:url, root: url, depth: 0, route: {url}} + visited[url] = true + + head = 1 + while head <= #queue + {url: feedUrl, :root, :depth, :route} = queue[head] + head += 1 + if depth >= maxDepth + record CrawlLimit.Depth, feedUrl, root, depth, route, maxDepth + continue + knownFeeds = @__loadKnownFeeds feedUrl + continue unless knownFeeds + stats.fetched += 1 + inventoryEntriesByUrl[feedUrl].fetched = true + + perFeedUntrusted = 0 + perFeedDrops, perRootDrops = newDrops!, newDrops! + for knownUrl in *knownFeeds + continue unless type(knownUrl) == "string" and #knownUrl > 0 + continue if @feedTrust\isBlocked knownUrl + trusted = @feedTrust\isTrusted knownUrl + unless trusted + -- bound how many untrusted feeds a single feed may contribute + if perFeedUntrusted >= maxPerFeed + drop perFeedDrops, knownUrl + continue + perFeedUntrusted += 1 + + -- a feed advertised by DepCtrl's own feed is official-known, not merely transitively known + prov = feedUrl == constants.DEPCTRL_FEED_URL and Provenance.OfficialKnown or Provenance.TransitiveKnown + entry = addSource inventoryEntriesByUrl, knownUrl, prov + entry.advertisedBy[feedUrl] = true + continue if visited[knownUrl] + + if trusted + visited[knownUrl] = true + queue[#queue + 1] = {url: knownUrl, :root, depth: depth + 1, route: appended(route, knownUrl)} + -- untrusted: within the per-root budget, ask the fetch policy before crawling in. shouldFetch + -- prompts under the `prompt` policy (session-cached), and denies under `never` or with no + -- prompter (headless), leaving the feed recorded above but not followed. + else + if (untrustedPerRoot[root] or 0) >= maxPerRoot + drop perRootDrops, knownUrl + elseif @feedTrust\shouldFetch knownUrl + untrustedPerRoot[root] = (untrustedPerRoot[root] or 0) + 1 + visited[knownUrl] = true + queue[#queue + 1] = {url: knownUrl, :root, depth: depth + 1, route: appended(route, knownUrl)} + + record CrawlLimit.PerFeed, feedUrl, root, depth, route, maxPerFeed, perFeedDrops if perFeedDrops.count > 0 + record CrawlLimit.PerRoot, feedUrl, root, depth, route, maxPerRoot, perRootDrops if perRootDrops.count > 0 + stats + +return FeedInventory diff --git a/modules/l0/DependencyControl/FeedLoader.moon b/modules/l0/DependencyControl/FeedLoader.moon new file mode 100644 index 0000000..5e32c8e --- /dev/null +++ b/modules/l0/DependencyControl/FeedLoader.moon @@ -0,0 +1,32 @@ +FileCache = require "l0.DependencyControl.FileCache" +constants = require "l0.DependencyControl.Constants" +Logger = require "l0.DependencyControl.Logger" +UpdateFeed = require "l0.DependencyControl.UpdateFeed" + +defaultLogger = Logger fileBaseName: "DepCtrl.FeedLoader" + +---Owns DependencyControl's single on-disk feed cache and hands out `UpdateFeed` instances wired to it, +---so no consumer assembles feed-fetch settings or names the cache. One instance is built per config (the +---updater keeps the shared one) and injected into every feed consumer; its cache is reachable as `.cache`. +---@class FeedLoader +---@field cache FileCache The shared feed cache, for metadata reads such as a feed's last-fetch time. +class FeedLoader + ---Reads the feed-fetch settings once and opens the shared feed cache under DepCtrl's namespace, wiring its + ---L1 layer to `UpdateFeed`'s decoder so a cache hit serves a ready-parsed feed base. + ---@param config ConfigView The global config view; reads `paths.cache`, `feeds.cacheMaxAge` and `updates.blockPrivateHosts`. + ---@param logger? Logger Logger for the cache and the feeds it loads. + new: (config, @logger = defaultLogger) => + c = config.c + @cache = FileCache.get c.paths.cache, constants.DEPCTRL_NAMESPACE, "feeds", + {maxAge: c.feeds.cacheMaxAge, logger: @logger, deserialize: UpdateFeed.deserialize} + @blockPrivateHosts = c.updates.blockPrivateHosts + + ---Builds an `UpdateFeed` for the given URL, injecting the shared cache and the configured fetch policy. + ---@param url string The feed URL to load. + ---@param opts? { autoLoad?: boolean } `autoLoad` fetches immediately (default true). To refresh an entire update pass, expire the cache with `FileCache.expireAll`. + ---@return UpdateFeed feed + load: (url, opts = {}) => + autoLoad = opts.autoLoad + autoLoad = true if autoLoad == nil + feedConfig = {cache: @cache, blockPrivateHosts: @blockPrivateHosts} + UpdateFeed url, autoLoad, nil, feedConfig, @logger diff --git a/modules/l0/DependencyControl/FeedManager.moon b/modules/l0/DependencyControl/FeedManager.moon new file mode 100644 index 0000000..06e157e --- /dev/null +++ b/modules/l0/DependencyControl/FeedManager.moon @@ -0,0 +1,130 @@ +constants = require "l0.DependencyControl.Constants" +Common = require "l0.DependencyControl.Common" +Crypto = require "l0.DependencyControl.Crypto" +Enum = require "l0.DependencyControl.Enum" +FeedInventory = require "l0.DependencyControl.FeedInventory" +FeedTrust = require "l0.DependencyControl.FeedTrust" + +FeedAction = Enum "FeedAction", { + Trust: "trust" + Block: "block" + Unblock: "unblock" + Remove: "remove" + OpenBrowser: "open-browser" +} + +-- The DepCtrl Browser pre-renders one page per crawled feed under this path, slugged by the feed URL's hash. +browserFeedBase = "https://typesettingtools.github.io/depctrl-browser/feeds/" + +---An action the Manage Feeds UI can offer for a feed. +---@alias FeedAction +---| "trust" # Trust: add the feed to the user's `trustedFeeds` +---| "block" # Block: add a block entry for the feed +---| "unblock" # Unblock: remove the user's exact block on the feed +---| "remove" # Remove: drop the user's `extraFeeds`/`trustedFeeds` contribution +---| "open-browser" # OpenBrowser: open the feed in the DepCtrl Browser + +---A Manage Feeds list row: a feed's inventory fields plus what the UI needs to render and act on it. +---@class FeedManagerRow +---@field url string The feed URL. +---@field trustStatus FeedTrustStatus The feed's trust state. +---@field inTrustedFeeds? boolean True when the feed is in the user's `trustedFeeds` (a trust-only listing). +---@field provenance FeedProvenance[] The sources the feed was found through. +---@field packages string[] Namespaces of installed packages tied to this feed. +---@field advertisedBy string[] Feeds that advertise this feed (crawl only). +---@field blockedBy? BlockedFeedEntry The matching block entry when the feed is blocked. +---@field reachable? boolean Whether a crawl fetched the feed (nil when not crawled). +---@field lastFetchedAt? integer Unix time the feed was last fetched into the persistent cache (nil if never cached). +---@field inUse? boolean Whether the feed is the effective update source of an installed package. +---@field actions FeedAction[] The actions offered for this feed. +---@field browserUrl string The feed's DepCtrl Browser deep-link. +---@field removable boolean Whether the feed has a user contribution that Remove can drop. + +---Turns reachable-feed data into what the Manage Feeds UI shows and offers: the actions available for a feed +---and its DepCtrl Browser link. Trust reads/writes go through `FeedTrust`. +---@class FeedManager +class FeedManager + @FeedAction = FeedAction + + ---@param feedTrust FeedTrust The trust model that action execution delegates to. + new: (@feedTrust) => + + ---The DepCtrl Browser deep-link for a feed. Best-effort: it resolves only for feeds in the browser's crawl + ---graph (official/known feeds). + ---@param feedUrl string The exact feed URL, as stored (the hash is sensitive to casing and trailing slash). + ---@return string url The Browser page URL. + @getBrowserUrl = (feedUrl) -> + "#{browserFeedBase}#{Crypto.sha1(feedUrl)\sub 1, 7}/" + + ---The actions the Manage Feeds UI offers for a feed, from its trust state and provenance. + ---@param entry FeedInventoryEntry The feed to compute the offered actions for. + ---@return FeedAction[] actions In a stable order, state-changing actions first and Open Browser last. + @getAvailableActions = (entry) -> + {:Provenance} = FeedInventory + {:TrustStatus} = FeedTrust + actions = {} + add = (action) -> actions[#actions + 1] = action + + add FeedAction.Trust if entry.trustStatus == TrustStatus.Untrusted + + add FeedAction.Block if entry.url != constants.DEPCTRL_FEED_URL and entry.trustStatus != TrustStatus.Blocked + + blk = entry.blockedBy + add FeedAction.Unblock if entry.trustStatus == TrustStatus.Blocked and blk and not blk.isOfficial and + blk.matchMode == FeedTrust.BlockMatchMode.Exact + + add FeedAction.Remove if Common.listIncludes(entry.provenance, Provenance.UserExtra) or entry.inTrustedFeeds + + add FeedAction.OpenBrowser + actions + + ---Turns feed entries into dialog rows — each feed with its offered actions, Browser link, and `removable` + ---flag. Sorted by URL. + ---@param entries FeedInventoryEntry[] The reachable-feed entries to render (from `FeedInventory.gather`/`crawl`). + ---@return FeedManagerRow[] rows The URL-sorted display rows. + @buildRows = (entries) -> + rows = {} + for entry in *entries + acts = FeedManager.getAvailableActions entry + rows[#rows + 1] = { + url: entry.url + trustStatus: entry.trustStatus + inTrustedFeeds: entry.inTrustedFeeds + provenance: entry.provenance + packages: entry.packages + advertisedBy: entry.advertisedBy + blockedBy: entry.blockedBy + reachable: entry.fetched + lastFetchedAt: entry.lastFetchedAt + inUse: entry.inUse + actions: acts + browserUrl: FeedManager.getBrowserUrl entry.url + removable: Common.listIncludes acts, FeedAction.Remove + } + table.sort rows, (a, b) -> a.url < b.url + rows + + ---Applies a trust action to a feed, delegating to `FeedTrust`. Re-checks that the action is one the feed + ---actually offers, so a guarded action (e.g. blocking the bootstrap feed) is refused even if asked for. + ---Open Browser and any action the feed doesn't offer mutate nothing. + ---@param action FeedAction The action to apply (one offered by `getAvailableActions`). + ---@param entry FeedInventoryEntry The feed to act on. + ---@param opts? { reason?: string } `reason` annotates a Block entry. + ---@return boolean applied Whether trust state was changed. + applyAction: (action, entry, opts = {}) => + return false unless Common.listIncludes @@.getAvailableActions(entry), action + switch action + when FeedAction.Trust + @feedTrust\trust entry.url + when FeedAction.Block + @feedTrust\block entry.url, {matchMode: FeedTrust.BlockMatchMode.Exact, reason: opts.reason} + when FeedAction.Unblock + @feedTrust\unblock entry.url + when FeedAction.Remove + @feedTrust\removeExtraFeed entry.url + @feedTrust\untrust entry.url + else + return false + true + +return FeedManager diff --git a/modules/l0/DependencyControl/FeedTrust.moon b/modules/l0/DependencyControl/FeedTrust.moon new file mode 100644 index 0000000..2c2b545 --- /dev/null +++ b/modules/l0/DependencyControl/FeedTrust.moon @@ -0,0 +1,347 @@ +constants = require "l0.DependencyControl.Constants" +Common = require "l0.DependencyControl.Common" +Enum = require "l0.DependencyControl.Enum" + +msgs = { + trustedFeedAdded: "Added '%s' to your trusted feeds." + trustedFeedRemoved: "Removed '%s' from your trusted feeds." + blockedFeedAdded: "Added '%s' to your blocked feeds." + blockedFeedRemoved: "Removed '%s' from your blocked feeds." + extraFeedAdded: "Added '%s' to your extra feeds." + extraFeedRemoved: "Removed '%s' from your extra feeds." +} + +---A normalized feed block entry: the URL/prefix to match, how to match it, and why. +---@class BlockedFeedEntry +---@field url string The feed URL or URL prefix to match. +---@field matchMode BlockedFeedMatchMode How `url` is matched against a candidate feed URL. +---@field reason? string Human-readable explanation of why the feed is blocked. +---@field isOfficial? boolean True for an entry from the official block list (read-only); false for a user block. + +---Owns DependencyControl's feed-trust model: the official trust lists (loaded from DepCtrl's own feed), +---the merged trusted/blocked sets (official plus the user's config), the trust queries the resolver asks +---per candidate, and the user-config mutations. Built and exposed by `Updater` as `updater.feedTrust`. +---@class FeedTrust +class FeedTrust + ---@alias BlockedFeedMatchMode + ---| "prefix" # Prefix: matches any feed URL starting with this value (case-insensitive) + ---| "exact" # Exact: matches only this exact feed URL (case-insensitive) + @BlockMatchMode = Enum "BlockedFeedMatchMode", { + Prefix: "prefix" + Exact: "exact" + } + + ---@alias FeedTrustStatus + ---| "trusted-official" # TrustedOfficial: trusted only through DependencyControl's official set + ---| "trusted-user" # TrustedUser: trusted only through one of the user's own lists (extraFeeds or trustedFeeds) + ---| "trusted-both" # TrustedBoth: trusted through both the official set and one of the user's own lists + ---| "untrusted" # Untrusted: neither trusted nor blocked (the default) + ---| "blocked" # Blocked: matched by the block list, which overrides trust + @TrustStatus = Enum "FeedTrustStatus", { + TrustedOfficial: "trusted-official" + TrustedUser: "trusted-user" + TrustedBoth: "trusted-both" + Untrusted: "untrusted" + Blocked: "blocked" + } + + ---@alias FeedFetchDecision + ---| "allow" # Allow: fetch without asking (trusted, or untrusted under fetchUntrustedFeeds = always) + ---| "deny" # Deny: never fetch (blocked, or untrusted under fetchUntrustedFeeds = never) + ---| "prompt" # Prompt: ask the user before fetching (untrusted under fetchUntrustedFeeds = prompt) + @FetchDecision = Enum "FeedFetchDecision", { + Allow: "allow" + Deny: "deny" + Prompt: "prompt" + } + + -- Default fetch policy for untrusted feeds, applied when the config key is unset. + ---@type FetchUntrustedFeeds + @defaultFetchUntrustedFeeds = Common.FetchUntrustedFeeds.Always + + ---@param config ConfigView The updater's config view; its `c` holds `extraFeeds`/`trustedFeeds`/`blockedFeeds`. + ---@param logger? Logger Logger for the trust/block confirmations. + ---@param feedLoader FeedLoader Loads DependencyControl's own feed for the official trust lists. + new: (@config, @logger, @feedLoader) => + + ---Lazily loads and caches DependencyControl's official trust lists from its own feed. Best-effort: if the + ---feed can't be loaded, only DepCtrl's own feed URL is trusted and nothing is blocked. + ---@private + ---@return { trusted: table, blocked: table[] } official Cached official trusted-feed set and raw block-list entries. + __loadOfficial: => + return @__official if @__official + trusted, blocked = {[constants.DEPCTRL_FEED_URL]: true}, {} + feed = @feedLoader\load constants.DEPCTRL_FEED_URL, {autoLoad: false} + if feed\ensureLoaded! + Common.makeSet feed\getKnownFeeds!, trusted + blocked = feed.data.blockedFeeds or {} + @__official = {:trusted, :blocked} + return @__official + + ---Returns the feed URLs DependencyControl officially trusts (its own feed URL plus the feeds it advertises). + ---@return table trustedFeeds + getOfficialTrustedFeeds: => @__loadOfficial!.trusted + + ---Returns DependencyControl's official block list as raw `{url, matchMode?, reason?}` entries, exactly as + ---declared in its own feed. + ---@return BlockedFeedEntry[] blockedFeeds + getOfficialBlockedFeeds: => @__loadOfficial!.blocked + + ---Returns the user's own trusted feeds: the union of `extraFeeds` (discovery roots) and `trustedFeeds` + ---(trust-only). Cached; invalidated when the user's lists change. + ---@return table userTrustedFeeds + getUserTrustedFeeds: => + unless @__userTrusted + c = @config.c.feeds + set = {} + Common.makeSet c.extraFeeds or {}, set + Common.makeSet c.trustedFeeds or {}, set + @__userTrusted = set + return @__userTrusted + + ---Returns the merged trusted feed-URL set: the official feeds plus the user's own trusted feeds + ---(`extraFeeds` and `trustedFeeds`). Cached; invalidated when the user's lists change. + ---@return table trustedFeeds + getTrustedFeeds: => + unless @__trusted + merged = {url, true for url in pairs @getOfficialTrustedFeeds!} + merged[url] = true for url in pairs @getUserTrustedFeeds! + @__trusted = merged + return @__trusted + + ---Returns a merged, normalized list of the "officially" blocked feeds (as per DependencyControls own feed), + ---followed by the user's blocked feeds. Block overrides trust. + ---@return BlockedFeedEntry[] blockedFeeds The merged list of normalized block entries, each tagged `isOfficial` when it comes from the official feed. + getBlockedFeeds: => + unless @__blocked + entries = {} + for raw in *@getOfficialBlockedFeeds! + entry = @@__normalizeBlockEntry raw + if entry + entry.isOfficial = true + entries[#entries + 1] = entry + for raw in *(@config.c.feeds.blockedFeeds or {}) + entry = @@__normalizeBlockEntry raw + if entry + entry.isOfficial = false + entries[#entries + 1] = entry + @__blocked = entries + return @__blocked + + ---Reports whether a feed URL is in the merged trusted set (exact match). + ---@param url? string The feed URL to check. + ---@return boolean trusted True when the feed URL is trusted, false otherwise. + isTrusted: (url) => url and @getTrustedFeeds![url] and true or false + + ---Reports whether a feed URL is trusted through one of the user's own lists (`extraFeeds` or `trustedFeeds`), + ---as opposed to DependencyControl's official set. A block does not factor into this. + ---@param url? string The feed URL to check. + ---@return boolean userTrusted True when the feed URL is in one of the user's trust lists. + isUserTrusted: (url) => url and @getUserTrustedFeeds![url] and true or false + + ---Reports whether a feed URL is in DependencyControl's official trusted set (its own feed plus the feeds it + ---advertises), as opposed to the user's own lists. A block does not factor into this. + ---@param url? string The feed URL to check. + ---@return boolean officiallyTrusted True when the feed URL is officially trusted. + isOfficiallyTrusted: (url) => url and @getOfficialTrustedFeeds![url] and true or false + + ---Reports whether a feed URL is matched by any block entry (official or user). + ---@param url? string The feed URL to check. + ---@return boolean blocked True when the feed URL is blocked, false otherwise. + isBlocked: (url) => @getBlockingEntry(url) != nil + + ---Returns the block entry that matches a feed URL or nil if none does. + ---Useful to get the reason for a block. + ---@param url? string The feed URL to check. + ---@return BlockedFeedEntry? entry The first matching block entry, or nil. + getBlockingEntry: (url) => + return nil unless url + for entry in *@getBlockedFeeds! + return entry if @@matchesBlockEntry url, entry + return nil + + ---Classifies a feed URL's trust: a block overrides any trust, and official vs user trust are reported + ---independently (a feed in both is `TrustedBoth`). + ---@param url? string The feed URL to classify. + ---@return FeedTrustStatus status + ---@return BlockedFeedEntry? blockingEntry The block entry that matched when the feed is blocked, else nil. + getTrustStatus: (url) => + blockingEntry = @getBlockingEntry url + return @@TrustStatus.Blocked, blockingEntry if blockingEntry + official = @isOfficiallyTrusted url + user = @isUserTrusted url + return @@TrustStatus.TrustedBoth if official and user + return @@TrustStatus.TrustedUser if user + return @@TrustStatus.TrustedOfficial if official + @@TrustStatus.Untrusted + + ---Classifies whether a feed may be fetched, from its block/trust status and the `fetchUntrustedFeeds` + ---policy: a blocked feed is always denied, a trusted feed always allowed, and an untrusted feed follows + ---the policy (always → allow, never → deny, prompt → prompt). Pure — it neither prompts nor mutates. + ---@param url? string The feed URL to classify. + ---@return FeedFetchDecision decision + getFetchDecision: (url) => + return @@FetchDecision.Deny if @isBlocked url + return @@FetchDecision.Allow if @isTrusted url + switch @config.c.feeds.fetchUntrustedFeeds or @@defaultFetchUntrustedFeeds + when Common.FetchUntrustedFeeds.Never then @@FetchDecision.Deny + when Common.FetchUntrustedFeeds.Prompt then @@FetchDecision.Prompt + else @@FetchDecision.Allow + + ---Resolves whether a feed may be fetched now, asking through the prompter (see setPrompter) when the + ---policy is `prompt`. A prompt answer is remembered for the session so the same feed isn't asked twice; + ---with no prompter available (e.g. headless), a `prompt` policy denies — the safe default. + ---@param url? string The feed URL to check. + ---@return boolean fetch True when the feed may be fetched. + shouldFetch: (url) => + switch @getFetchDecision url + when @@FetchDecision.Allow then true + when @@FetchDecision.Deny then false + else @__resolvePrompt url + + ---Sets the callback consulted for an untrusted feed under the `prompt` policy. It receives the feed URL + ---and this FeedTrust (so it may trust/block the feed) and returns whether to fetch it now. + ---@param prompter? fun(url: string, feedTrust: FeedTrust): boolean The prompter, or nil to remove it. + setPrompter: (@prompter) => + + ---@private + ---@param url string + ---@return boolean fetch + __resolvePrompt: (url) => + @__promptAnswers or= {} + answer = @__promptAnswers[url] + return answer unless answer == nil + prompter = @prompter + answer = prompter and prompter(url, @) and true or false + @__promptAnswers[url] = answer + return answer + + ---Appends a feed URL to one of the user's config lists (skipping an exact duplicate), invalidates the + ---cached trusted set, and persists. + ---@private + ---@param configKey string The user config array field ("trustedFeeds" or "extraFeeds"). + ---@param feedUrl string The exact feed URL to add. + ---@return boolean added True when the URL was added, false when it was already present. + __addUserFeed: (configKey, feedUrl) => + list = [url for url in *(@config.c.feeds[configKey] or {})] + return false if Common.listIncludes list, feedUrl + list[#list + 1] = feedUrl + @config.c.feeds[configKey] = list + @__trusted, @__userTrusted = nil, nil + @config\save! + return true + + ---Removes every exact match of a feed URL from one of the user's config lists, invalidates the cached + ---trusted set, and persists when something changed. + ---@private + ---@param configKey string The user config array field ("trustedFeeds" or "extraFeeds"). + ---@param feedUrl string The exact feed URL to remove. + ---@return boolean removed True when a feed was removed from the user's config, false when no entry matched. + __removeUserFeed: (configKey, feedUrl) => + list = @config.c.feeds[configKey] or {} + kept = [url for url in *list when url != feedUrl] + return false if #kept == #list + @config.c.feeds[configKey] = kept + @__trusted, @__userTrusted = nil, nil + @config\save! + return true + + ---Adds a feed URL to the user's `trustedFeeds` list in the DependencyControl config file (ignoring an exact duplicate). + ---@param feedUrl string The exact (case-sensitive) feed URL to trust. + ---@return boolean added True when the feed was added to the user's `trustedFeeds`, false when it was already present. + trust: (feedUrl) => + added = @__addUserFeed "trustedFeeds", feedUrl + @logger\log msgs.trustedFeedAdded, feedUrl if added and @logger + return added + + ---Removes a feed URL from the user's `trustedFeeds` list in the DependencyControl config file. + ---Feeds trusted through the official list or `extraFeeds` are unaffected; block the feed to override those. + ---@param feedUrl string The exact (case-sensitive) feed URL to untrust. + ---@return boolean removed True, when the feed was removed from the user's `trustedFeeds`, false when it wasn't present. + untrust: (feedUrl) => + removed = @__removeUserFeed "trustedFeeds", feedUrl + @logger\log msgs.trustedFeedRemoved, feedUrl if removed and @logger + return removed + + ---Adds a new entry to the user's `blockedFeeds` in the DependencyControl config file, + ---unless a block with the same url and match mode is already present. + ---@param feedUrl string The feed URL or URL prefix to block. + ---@param opts? { matchMode?: BlockedFeedMatchMode, reason?: string } The match mode (default prefix) and an optional reason. + ---@return boolean added False when an equivalent block (same url and match mode) was already present. + block: (feedUrl, opts = {}) => + matchMode = opts.matchMode or @@BlockMatchMode.Prefix + entries = [e for e in *(@config.c.feeds.blockedFeeds or {})] + for raw in *entries + norm = @@__normalizeBlockEntry raw + return false if norm and norm.url == feedUrl and norm.matchMode == matchMode + entries[#entries + 1] = {url: feedUrl, :matchMode, reason: opts.reason} + @config.c.feeds.blockedFeeds = entries + @__blocked = nil + @config\save! + @logger\log msgs.blockedFeedAdded, feedUrl if @logger + return true + + ---Removes every entry from the user's `blockedFeeds` list in the DependencyControl config file, + ---whose url matches `feedUrl`. Does not affect the official block list. + ---@param feedUrl string The blocked url/prefix to remove. + ---@return boolean removed True when a user block was removed, false when no user block matched. + unblock: (feedUrl) => + list = @config.c.feeds.blockedFeeds or {} + kept = [raw for raw in *list when (@@__normalizeBlockEntry(raw) or {}).url != feedUrl] + return false if #kept == #list + @config.c.feeds.blockedFeeds = kept + @__blocked = nil + @config\save! + @logger\log msgs.blockedFeedRemoved, feedUrl if @logger + return true + + ---Adds a feed URL to the user's `extraFeeds` config (ignoring an exact duplicate) and persists it. Extra + ---feeds are trusted and act as discovery roots. + ---@param feedUrl string The exact (case-sensitive) feed URL to add. + ---@return boolean added False when the feed was already in the user's `extraFeeds`. + addExtraFeed: (feedUrl) => + added = @__addUserFeed "extraFeeds", feedUrl + @logger\log msgs.extraFeedAdded, feedUrl if added and @logger + return added + + ---Removes a feed URL from the user's `extraFeeds` config and persists it. + ---@param feedUrl string The exact (case-sensitive) feed URL to remove. + ---@return boolean removed False when the feed was not in the user's `extraFeeds`. + removeExtraFeed: (feedUrl) => + removed = @__removeUserFeed "extraFeeds", feedUrl + @logger\log msgs.extraFeedRemoved, feedUrl if removed and @logger + return removed + + ---Reports whether a URL is matched by any of the given prefixes. + ---Case-insensitive to resist evasion attempts. + ---@param url? string The feed URL to check. + ---@param prefixes? string[] The list of prefixes to match against. + ---@return boolean matches True when the feed URL matches any of the prefixes, false otherwise. + @urlMatchesPrefix = (url, prefixes = {}) => + return false unless url + url = url\lower! + for prefix in *prefixes + prefix = prefix\lower! + return true if #prefix > 0 and url\sub(1, #prefix) == prefix + return false + + ---Normalizes a raw block table into a `BlockedFeedEntry`, applying defaults as per the schema. + ---@private + ---@param entry { url, matchMode?, reason? } A raw entry from a feed's or the user's `blockedFeeds`. + ---@return BlockedFeedEntry? normalized The normalized entry, or nil when it carries no url. + @__normalizeBlockEntry = (entry) => + return nil unless type(entry) == "table" and type(entry.url) == "string" + matchMode = @BlockMatchMode\validate(entry.matchMode) and entry.matchMode or @BlockMatchMode.Prefix + return {url: entry.url, :matchMode, reason: entry.reason} + + ---Reports whether a URL is matched by a single block entry, per its match mode. + ---All matches are case-insensitive to combat evasion attempts. + ---@param url? string The feed URL to check. + ---@param entry BlockedFeedEntry A normalized block entry. + ---@return boolean matches True when the feed URL is blocked, false otherwise. + @matchesBlockEntry = (url, entry) => + return false unless url and entry and entry.url + return url\lower! == entry.url\lower! if entry.matchMode == @BlockMatchMode.Exact + return @urlMatchesPrefix url, {entry.url} + +return FeedTrust diff --git a/modules/l0/DependencyControl/FileCache.moon b/modules/l0/DependencyControl/FileCache.moon new file mode 100644 index 0000000..53fb043 --- /dev/null +++ b/modules/l0/DependencyControl/FileCache.moon @@ -0,0 +1,253 @@ +Crypto = require "l0.DependencyControl.Crypto" +FileOps = require "l0.DependencyControl.FileOps" +Logger = require "l0.DependencyControl.Logger" +Lock = require "l0.DependencyControl.Lock" +dkjson = require "l0.dkjson" + +defaultLogger = Logger fileBaseName: "DepCtrl.FileCache" + +LOCK_NAMESPACE = "l0.DependencyControl.FileCache" -- Global-lock namespace for serializing cache writes +LOCK_TIMEOUT = 5000 -- ms to wait for the write lock before skipping the write + +-- Replaces filesystem-hostile characters so a cache entry's label is safe in a file name, and clamps its +-- length. Falls back to "entry" for an empty or missing label. +sanitizeLabel = (label) -> + safe = tostring(label or "entry")\gsub "[^%w%._-]", "_" + #safe > 0 and safe\sub(1, 64) or "entry" + +-- The 7-hex-char SHA-1 slug of a cache key. Deterministic per key (sensitive to its exact bytes), matching +-- the DepCtrl Browser's feed-URL slug convention. +keySlug = (key) -> Crypto.sha1(key)\sub 1, 7 + +-- The embedded UTC timestamp of a snapshot file name, for chronological ordering ("" when absent). +snapshotStamp = (fileName) -> fileName\match "(%d+T%d+Z)" or "" + +-- An instance's on-disk directory: the configured base, namespaced and named. The single place the layout +-- is defined, shared by the constructor and the `get` factory's registry key. +resolveDir = (basePath, namespace, name) -> "#{aegisub.decode_path basePath}/#{namespace}/#{name}" + +---The per-key index entry FileCache persists next to each snapshot; returned by getMeta/getFile/get/put. +---@class FileCacheMeta +---@field key string The cache key this entry indexes. +---@field cachedAt integer Unix time the latest snapshot was written. +---@field expiresAt integer Unix time the entry turns stale, fixed at write time. +---@field latestFile string File name of the latest snapshot, relative to the cache directory. + +---Construction options for FileCache. +---@class FileCacheOptions +---@field maxAge? integer Default entry lifetime in seconds, used when a put doesn't set its own (default 3600). +---@field maxFiles? integer Snapshot files retained per cache before the oldest are trimmed (default 50). +---@field logger? Logger Logger for cache operations. +---@field now? fun(): integer Clock override returning Unix time; defaults to os.time (injected in tests). +---@field deserialize? fun(content: string): any Codec turning stored content into the value get returns and memoizes; its presence enables the in-memory L1 layer. + +---A persistent, key-addressed cache of JSON blobs on disk. Each `put` is stored as a human-readable, +---timestamped snapshot file (which doubles as a history), indexed by a deterministic per-key meta file that +---names the latest snapshot and records when it was cached. An instance lives under +---`//`, so distinct caches (feeds, and others later) share one configurable cache +---base without colliding. Prefer the `get` factory over the constructor so instances are shared per directory. +---With a `deserialize` codec, `get` adds an in-memory L1 layer over the on-disk L2: it returns the +---parsed snapshot and memoizes it, keyed to the snapshot's cache time so a newer `put` (here or in another +---process) transparently supersedes the memo. +---@class FileCache +class FileCache + @defaultMaxAge = 3600 -- default entry lifetime (seconds), fixed into each entry's index at write time + @defaultMaxFiles = 50 -- how many snapshot files to retain before trimming the oldest + + -- Shared instances keyed by resolved directory, so callers for the same cache get one instance (and thus + -- share its L1 memo). Weak values let an unreferenced cache be collected while a live one stays shared. + @__instances = setmetatable {}, __mode: "v" + + ---Returns the shared cache for a base/namespace/name, reusing the existing instance for that resolved + ---directory rather than constructing a duplicate. Options apply only when the instance is first created. + ---@param basePath string The cache root (the `paths.cache` setting, e.g. "?user/cache"); path-decoded. + ---@param namespace string The owning script namespace (`constants.DEPCTRL_NAMESPACE` for DepCtrl's own caches). + ---@param name string A short name for this cache's purpose (e.g. "feeds"). + ---@param opts? FileCacheOptions See new. + ---@return FileCache + @get = (basePath, namespace, name, opts) -> + dir = resolveDir basePath, namespace, name + cache = FileCache.__instances[dir] + unless cache + cache = FileCache basePath, namespace, name, opts + FileCache.__instances[dir] = cache + return cache + + ---@param basePath string The cache root (the `paths.cache` setting, e.g. "?user/cache"); path-decoded here. + ---@param namespace string The owning script namespace (`constants.DEPCTRL_NAMESPACE` for DepCtrl's own caches). + ---@param name string A short subdirectory naming this cache's purpose (e.g. "feeds"). + ---@param opts? FileCacheOptions Defaults for entry lifetime, retention, logging, clock, and the L1 codec. + new: (basePath, namespace, name, opts = {}) => + @cacheDir = resolveDir basePath, namespace, name + @maxAge = opts.maxAge or @@defaultMaxAge + @maxFiles = opts.maxFiles or @@defaultMaxFiles + @logger = opts.logger or defaultLogger + @now = opts.now or os.time + @__deserialize = opts.deserialize + -- L1: key -> {value, cachedAt}; each memo mirrors one L2 snapshot and is superseded when its cachedAt does + @__l1 = {} + + ---@private + __metaPath: (key) => FileOps.joinPath @cacheDir, "#{keySlug key}.meta.json" + + ---Reads and decodes a cache index (meta) JSON file. + ---@private + ---@param path string + ---@return FileCacheMeta? meta + __readMeta: (path) => + content = FileOps.readFile path + return nil unless content + -- a torn read from a concurrent write fails to decode and is treated as a cache miss + ok, meta = pcall dkjson.decode, content + ok and type(meta) == "table" and meta or nil + + ---Reads the cache index entry for a key. + ---@param key string + ---@return FileCacheMeta? meta nil when the key isn't cached. + getMeta: (key) => @__readMeta @__metaPath key + + ---Whether a cache entry is still fresh, i.e. its expiry (fixed when it was written) hasn't passed + ---and it hasn't been invalidated by expireAll. The expiry is intrinsic to the entry, so freshness is the + ---same regardless of which instance asks or what its current `maxAge` is. + ---@param meta FileCacheMeta? A meta returned by getMeta. + ---@return boolean fresh + isFresh: (meta) => + return false unless meta and meta.expiresAt + return false if @__staleBefore and meta.cachedAt and meta.cachedAt < @__staleBefore + @.now! < meta.expiresAt + + ---Resolves the latest cached snapshot for a key. The snapshot may be stale; callers use isFresh on the + ---returned meta to decide whether to serve it directly or only as an offline fallback. + ---@param key string + ---@return string? path The snapshot file's path, or nil when the key isn't cached (or its file is gone). + ---@return FileCacheMeta? meta The cache index entry (returned even when the snapshot file is missing). + getFile: (key) => + meta = @getMeta key + return nil unless meta and meta.latestFile + path = FileOps.joinPath @cacheDir, meta.latestFile + return nil, meta unless "file" == FileOps.attributes path, "mode" + return path, meta + + ---Returns the deserialized latest snapshot for a key, served from the in-memory L1 memo when it still + ---mirrors the on-disk snapshot, otherwise read and deserialized from L2 and memoized. The value may be + ---stale — freshness is reported separately so the caller can serve it or treat it as an offline fallback. + ---Requires a `deserialize` codec; without one this always misses. + ---@param key string + ---@return any? value The deserialized snapshot (the freshest available, even when stale), or nil if not cached. + ---@return FileCacheMeta? meta The snapshot's index entry, returned even when stale. + ---@return boolean fresh Whether the returned snapshot is still within its expiry. + get: (key) => + meta = @getMeta key + return nil unless meta and meta.cachedAt + fresh = @isFresh meta + memo = @__l1[key] + return memo.value, meta, fresh if memo and memo.cachedAt == meta.cachedAt + return nil, meta, fresh unless @__deserialize and meta.latestFile + + path = FileOps.joinPath @cacheDir, meta.latestFile + content = "file" == FileOps.attributes(path, "mode") and FileOps.readFile path + return nil, meta, fresh unless content + + value = @.__deserialize content + @__l1[key] = {:value, cachedAt: meta.cachedAt} + return value, meta, fresh + + ---Marks every entry cached before the cut-off as stale, so the next get/isFresh reports it not fresh. The + ---snapshot is kept as an offline fallback, and a later put makes the entry fresh again. When purging, the + ---affected entries are instead dropped from the L1 memo and their on-disk snapshots and index deleted, to + ---reclaim space or hard-reset. + ---@param before? integer Cut-off Unix time; entries cached strictly before it are affected (default: now). + ---@param purge? boolean Delete the affected entries (L1 + L2) instead of only marking them stale (default false). + expireAll: (before = @.now!, purge = false) => + @__staleBefore = before + return unless purge + + files = FileOps.listDir @cacheDir + return unless files + + -- collect the key slugs whose index predates the cut-off, dropping their memos as we go + expiredSlugs = {} + for file in *files + continue unless file\match "%.meta%.json$" + meta = @__readMeta FileOps.joinPath @cacheDir, file + continue unless meta and meta.cachedAt and meta.cachedAt < before + expiredSlugs[keySlug meta.key] = true + @__l1[meta.key] = nil + + -- every file (snapshot or index) is named with its key's 7-hex slug prefix, so one pass removes both + for file in *files + slug = file\match "^(%x%x%x%x%x%x%x)" + FileOps.remove FileOps.joinPath @cacheDir, file if slug and expiredSlugs[slug] + + ---Stores a blob under a readable, timestamped snapshot and repoints the index at it, then trims old + ---snapshots. The snapshot name is `-