fix(KAN-65): allow rebuilding unpublished macOS releases

.github/workflows/macos-release.yml

@@ -17,6 +17,8 @@ permissions:
   contents: read
 
 concurrency:
+  # Partial release files may be replaced until the feed is promoted.
+  # Keep every publisher in this group to prevent overlapping rebuilds.
   group: macos-release
   cancel-in-progress: false
 
@@ -198,7 +200,7 @@ jobs:
         id: package
         run: node native/macos-ci.mjs package
 
-      - name: Publish immutable files and verify public Hetzner downloads
+      - name: Upload release files and promote the verified update feed
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}

native/publish-macos.mjs

@@ -73,7 +73,7 @@ export function conditionalEtag(etag) {
         !/^[a-f\d]{32}(?:-[1-9]\d*)?$/i.test(value)
     ) {
         throw new Error(
-            'Cannot atomically update the feed without a valid S3 ETag.',
+            'Cannot atomically replace an object without a valid S3 ETag.',
         );
     }
     // Hetzner returns quoted ETags on GET but compares bare values on conditional PUT.
@@ -185,12 +185,15 @@ export async function publishMacos({
     const staging = mkdtempSync(join(tmpdir(), 'kanbangit-s3-'));
     const keyPrefix = `${storage.prefix}/${metadata.target}/${metadata.version}`;
     const publicPrefix = `${storage.publicBase}/${keyPrefix}`;
+    const latestPrefix = `${storage.prefix}/${metadata.target}`;
+    const latestKey = `${latestPrefix}/latest.json`;
     async function verifyPublic(name, contents, prefix = publicPrefix) {
         for (let attempt = 1; attempt <= 5; attempt++) {
             let response;
             try {
                 response = await fetchPublic(`${prefix}/${name}`, {
                     redirect: 'error',
+                    cache: 'no-store',
                     signal: AbortSignal.timeout(60000),
                 });
             } catch (error) {
@@ -225,36 +228,86 @@ export async function publishMacos({
             await wait(2000);
         }
     }
-    function putImmutable(name, contents, contentType) {
+    function assertUnpublishedRelease() {
+        const file = join(staging, 'published-latest.json');
+        try {
+            s3('get-object', ['--key', latestKey, file]);
+        } catch (error) {
+            if (/\(NoSuchKey\)/.test(error.message)) {
+                return;
+            }
+            if (requiresExistingObjectCheck(error)) {
+                // A broken CLI error message cannot distinguish missing from denied.
+                // Only an authenticated, explicitly empty listing proves absence.
+                const listing = JSON.parse(
+                    s3('list-objects-v2', [
+                        '--prefix',
+                        latestKey,
+                        '--max-keys',
+                        '1',
+                        '--output',
+                        'json',
+                    ]),
+                );
+                if (
+                    listing?.KeyCount === 0 &&
+                    listing.IsTruncated === false &&
+                    Array.isArray(listing.Contents ?? []) &&
+                    (listing.Contents ?? []).length === 0
+                ) {
+                    return;
+                }
+            }
+            throw error;
+        }
+        const published = JSON.parse(readFileSync(file, 'utf8'));
+        if (compareVersions(metadata.version, published.version) <= 0) {
+            throw new Error(
+                'This version has already been released through the update feed; bump the app version. Published objects were not overwritten.',
+            );
+        }
+    }
+    // The macos-release concurrency group serializes the entire publication.
+    // Never run independent publishers against the same release destination.
+    function putReleaseFile(name, contents, contentType) {
         const file = join(staging, name);
         writeFileSync(file, contents);
-        try {
+        function put(condition) {
             s3('put-object', [
                 '--key',
                 `${keyPrefix}/${name}`,
                 '--body',
                 file,
-                '--if-none-match',
+                ...condition,
-                '*',
                 '--content-type',
                 contentType,
                 '--cache-control',
-                'public, max-age=31536000, immutable',
+                'no-cache, max-age=0, must-revalidate',
                 '--content-md5',
                 createHash('md5').update(contents).digest('base64'),
                 '--metadata',
                 `sha256=${sha256(contents)}`,
             ]);
+        }
+        try {
+            put(['--if-none-match', '*']);
         } catch (error) {
             if (!requiresExistingObjectCheck(error)) {
                 throw error;
             }
             const existing = join(staging, `existing-${name}`);
-            s3('get-object', ['--key', `${keyPrefix}/${name}`, existing]);
+            const response = JSON.parse(
-            if (sha256(readFileSync(existing)) !== sha256(contents)) {
+                s3('get-object', [
-                throw new Error(
+                    '--key',
-                    `Published ${name} differs; bump the app version. Existing objects were not overwritten.`,
+                    `${keyPrefix}/${name}`,
+                    '--output',
+                    'json',
+                    existing,
+                ]),
             );
+            if (sha256(readFileSync(existing)) !== sha256(contents)) {
+                assertUnpublishedRelease();
+                put(['--if-match', conditionalEtag(response.ETag)]);
             }
         }
     }
@@ -279,7 +332,7 @@ export async function publishMacos({
             ],
         ];
         for (const [name, contents, type] of files) {
-            putImmutable(name, contents, type);
+            putReleaseFile(name, contents, type);
             await verifyPublic(name, contents);
         }
         // Completion metadata is written only after every public artifact matches.
@@ -296,7 +349,7 @@ export async function publishMacos({
                 2,
             ) + '\n',
         );
-        putImmutable('release.json', manifest, 'application/json');
+        putReleaseFile('release.json', manifest, 'application/json');
         await verifyPublic('release.json', manifest);
         const latest = Buffer.from(
             JSON.stringify(
@@ -310,10 +363,8 @@ export async function publishMacos({
                 2,
             ) + '\n',
         );
-        putImmutable('update.json', latest, 'application/json');
+        putReleaseFile('update.json', latest, 'application/json');
         await verifyPublic('update.json', latest);
-        const latestPrefix = `${storage.prefix}/${metadata.target}`;
-        const latestKey = `${latestPrefix}/latest.json`;
         const latestFile = join(staging, 'latest.json');
         writeFileSync(latestFile, latest);
         function publishLatest(condition) {

native/tests/macos-workflow.test.js

@@ -75,7 +75,7 @@ function fixture(t) {
         );
         return { ...result, output: readFileSync(output, 'utf8') };
     };
-    return { run, git, checkout, commit };
+    return { run, git, checkout, remote, commit };
 }
 
 test('release selection succeeds using Git alone without an administrative API token', (t) => {
@@ -117,6 +117,35 @@ test('manual release on main accepts an existing tag without moving it', (t) =>
     assert.match(workflow, /node native\/publish-macos.mjs upload/);
 });
 
+test('a moved macOS tag selects the new main commit for a full rebuild', (t) => {
+    const f = fixture(t);
+    f.git(
+        f.remote,
+        '-c',
+        'commit.gpgsign=false',
+        'commit',
+        '--allow-empty',
+        '-m',
+        'Fix failed release',
+    );
+    const rebuilt = f.git(f.remote, 'rev-parse', 'HEAD');
+    f.git(f.remote, 'tag', '--force', 'macos-v1.2.3');
+    f.git(
+        f.checkout,
+        'fetch',
+        '--force',
+        'origin',
+        'refs/tags/macos-v1.2.3:refs/tags/macos-v1.2.3',
+    );
+    const result = f.run();
+    assert.equal(result.status, 0, result.stderr);
+    assert.equal(result.output, `commit=${rebuilt}\ntag=macos-v1.2.3\n`);
+    assert.notEqual(rebuilt, f.commit);
+    assert.match(workflow, /tags:\n\s+- "macos-v\*"/);
+    assert.match(workflow, /^ {2}group: macos-release$/m);
+    assert.match(workflow, /^ {2}cancel-in-progress: false$/m);
+});
+
 test('release DMG packaging disables Finder automation in the Tauri action environment', () => {
     const build = workflow
         .split('      - name: Build, sign and notarize with Tauri\n')[1]

native/tests/publish-macos.test.js

@@ -99,7 +99,7 @@ function fixture(
         const operation = args[args.indexOf('s3api') + 1];
         const value = (name) => args[args.indexOf(name) + 1];
         const key = value('--key');
-        events.push({ operation, key });
+        events.push({ operation, key, args });
         if (operation === 'get-bucket-versioning') {
             assert.equal(value('--output'), 'json');
             return '';
@@ -121,6 +121,9 @@ function fixture(
             return '{}';
         }
         if (operation === 'get-object') {
+            if (!objects.has(key)) {
+                throw new Error('aws failed: (NoSuchKey) object missing');
+            }
             writeFileSync(args.at(-1), objects.get(key));
             return JSON.stringify({
                 ETag: '"b88312b96169f4b2098a4cc43b0abb6a"',
@@ -156,7 +159,7 @@ function fixture(
     };
 }
 
-test('uploads immutable files and writes completion record after public verification', async (t) => {
+test('uploads revalidated files and writes completion record after public verification', async (t) => {
     const f = fixture(t);
     const url = await publishMacos(f);
     assert.equal(
@@ -168,6 +171,14 @@ test('uploads immutable files and writes completion record after public verifica
         8,
     );
     assert.equal(f.objects.size, 8);
+    for (const event of f.events.filter(
+        (event) => event.operation === 'put-object',
+    )) {
+        assert.equal(
+            event.args[event.args.indexOf('--cache-control') + 1],
+            'no-cache, max-age=0, must-revalidate',
+        );
+    }
     assert(f.events.at(-1).key.endsWith('/latest.json'));
     assert.equal(
         f.events.at(-1).key,
@@ -230,14 +241,20 @@ test('retry accepts byte-identical objects without replacing them', async (t) =>
     );
 });
 
-test('different existing release is never overwritten', async (t) => {
+for (const published of ['1.2.3', '1.2.4']) {
+    test(`published release stays protected when the feed is at ${published}`, async (t) => {
         const f = fixture(t);
         const key = `releases/macos/${f.metadata.target}/1.2.3/${f.metadata.filename}`;
         f.objects.set(key, Buffer.from('previous version content'));
+        f.objects.set(
+            `releases/macos/${f.metadata.target}/latest.json`,
+            Buffer.from(JSON.stringify({ version: published })),
+        );
         await assert.rejects(publishMacos(f), /bump the app version/);
         assert.equal(f.objects.get(key).toString(), 'previous version content');
-    assert.equal(f.objects.size, 1);
+        assert.equal(f.objects.size, 2);
     });
+}
 
 test('checksum mismatch aborts before contacting S3', async (t) => {
     const f = fixture(t);
@@ -292,7 +309,7 @@ for (const failure of ['private', 'wrong-content', 'network']) {
     });
 }
 
-test('a partial upload can resume with the same files', async (t) => {
+test('a partial upload can be retried with the same files', async (t) => {
     const f = fixture(t);
     const run = f.run;
     f.run = (command, args) => {
@@ -311,6 +328,203 @@ test('a partial upload can resume with the same files', async (t) => {
     assert.equal(f.objects.size, 8);
 });
 
+function rebuild(f) {
+    const updater = Buffer.from('rebuilt signed update');
+    const signing = signedFixture(updater);
+    f.publicKey = signing.publicKey;
+    f.metadata.notes = 'Fixed build of the same version';
+    for (const [name, contents] of [
+        [f.metadata.filename, Buffer.from('rebuilt disk image')],
+        [f.metadata.updaterFilename, updater],
+    ]) {
+        writeFileSync(join(f.directory, name), contents);
+        writeFileSync(
+            join(f.directory, `${name}.sha256`),
+            `${createHash('sha256').update(contents).digest('hex')}  ${name}\n`,
+        );
+    }
+    writeFileSync(
+        join(f.directory, `${f.metadata.updaterFilename}.sig`),
+        signing.signature,
+    );
+}
+
+for (const previous of [null, '1.0.0']) {
+    for (const interruptedFile of ['.sha256', '/latest.json']) {
+        test(`rebuild replaces unfinished ${interruptedFile} release with previous feed ${previous}`, async (t) => {
+            const f = fixture(t);
+            const latestKey = `releases/macos/${f.metadata.target}/latest.json`;
+            const previousBytes =
+                previous === null
+                    ? undefined
+                    : Buffer.from(JSON.stringify({ version: previous }));
+            if (previousBytes) {
+                f.objects.set(latestKey, previousBytes);
+            }
+            const run = f.run;
+            f.run = (command, args) => {
+                if (
+                    args.includes('put-object') &&
+                    args[args.indexOf('--key') + 1].endsWith(interruptedFile)
+                ) {
+                    throw new Error('upload interrupted');
+                }
+                return run(command, args);
+            };
+            await assert.rejects(publishMacos(f), /upload interrupted/);
+            assert.deepEqual(f.objects.get(latestKey), previousBytes);
+            rebuild(f);
+            f.events.length = 0;
+            let promoted = false;
+            f.run = (command, args) => {
+                if (!promoted) {
+                    assert.deepEqual(f.objects.get(latestKey), previousBytes);
+                }
+                const result = run(command, args);
+                if (args.includes('put-object') && args.includes(latestKey)) {
+                    promoted = true;
+                }
+                return result;
+            };
+            await publishMacos(f);
+            assert.equal(f.objects.size, 8);
+            assert.equal(JSON.parse(f.objects.get(latestKey)).version, '1.2.3');
+            assert.equal(
+                JSON.parse(f.objects.get(latestKey)).notes,
+                f.metadata.notes,
+            );
+            for (const name of [
+                f.metadata.filename,
+                `${f.metadata.filename}.sha256`,
+                f.metadata.updaterFilename,
+                `${f.metadata.updaterFilename}.sig`,
+                `${f.metadata.updaterFilename}.sha256`,
+            ]) {
+                assert.deepEqual(
+                    f.objects.get(
+                        `releases/macos/${f.metadata.target}/1.2.3/${name}`,
+                    ),
+                    readFileSync(join(f.directory, name)),
+                );
+            }
+            const published = [...f.objects.entries()];
+            rebuild(f);
+            await assert.rejects(publishMacos(f), /already been released/);
+            assert.deepEqual([...f.objects.entries()], published);
+        });
+    }
+}
+
+test('failure after promoting the feed still protects that version from rebuilding', async (t) => {
+    const f = fixture(t);
+    const fetchPublic = f.fetchPublic;
+    f.fetchPublic = async (url, options) =>
+        url.endsWith('/latest.json')
+            ? { ok: false, status: 403 }
+            : fetchPublic(url, options);
+    await assert.rejects(publishMacos(f), /HTTP 403/);
+    const published = [...f.objects.entries()];
+    rebuild(f);
+    await assert.rejects(publishMacos(f), /already been released/);
+    assert.deepEqual([...f.objects.entries()], published);
+});
+
+test('a concurrent change to a partial artifact is not overwritten', async (t) => {
+    const f = fixture(t);
+    const key = `releases/macos/${f.metadata.target}/1.2.3/${f.metadata.filename}`;
+    f.objects.set(key, Buffer.from('unfinished old build'));
+    const run = f.run;
+    f.run = (command, args) => {
+        if (args.includes('--if-match')) {
+            assert(args.includes(key));
+            f.objects.set(key, Buffer.from('concurrent winner'));
+            throw new Error('(PreconditionFailed) concurrent replacement');
+        }
+        return run(command, args);
+    };
+    await assert.rejects(publishMacos(f), /concurrent replacement/);
+    assert.equal(f.objects.get(key).toString(), 'concurrent winner');
+    assert.equal(f.objects.size, 1);
+});
+
+for (const failure of [
+    'AccessDenied',
+    'network failure',
+    'invalid JSON',
+    'invalid version',
+]) {
+    test(`cannot replace a partial artifact when the feed check has ${failure}`, async (t) => {
+        const f = fixture(t);
+        const key = `releases/macos/${f.metadata.target}/1.2.3/${f.metadata.filename}`;
+        const latestKey = `releases/macos/${f.metadata.target}/latest.json`;
+        f.objects.set(key, Buffer.from('unfinished old build'));
+        f.objects.set(
+            latestKey,
+            Buffer.from(
+                failure === 'invalid version' ? '{"version":"invalid"}' : '{',
+            ),
+        );
+        const originals = [...f.objects.entries()];
+        const run = f.run;
+        f.run = (command, args) => {
+            if (
+                args.includes('get-object') &&
+                args.includes(latestKey) &&
+                !failure.startsWith('invalid')
+            ) {
+                throw new Error(failure);
+            }
+            return run(command, args);
+        };
+        await assert.rejects(publishMacos(f));
+        assert.deepEqual([...f.objects.entries()], originals);
+    });
+}
+
+for (const listing of [
+    { KeyCount: 0, IsTruncated: false },
+    { KeyCount: 1, IsTruncated: false, Contents: [{ Key: 'latest.json' }] },
+    {},
+    { KeyCount: 0, IsTruncated: true },
+    { KeyCount: 0, IsTruncated: false, Contents: '' },
+    'AccessDenied',
+]) {
+    test(`ambiguous feed read error needs a proven empty listing: ${JSON.stringify(listing)}`, async (t) => {
+        const f = fixture(t);
+        const key = `releases/macos/${f.metadata.target}/1.2.3/${f.metadata.filename}`;
+        const latestKey = `releases/macos/${f.metadata.target}/latest.json`;
+        f.objects.set(key, Buffer.from('unfinished old build'));
+        const run = f.run;
+        f.run = (command, args) => {
+            if (args.includes('get-object') && args.includes(latestKey)) {
+                throw new Error(
+                    "argument of type 'NoneType' is not a container or iterable",
+                );
+            }
+            if (args.includes('list-objects-v2')) {
+                assert.equal(args[args.indexOf('--prefix') + 1], latestKey);
+                if (listing === 'AccessDenied') {
+                    throw new Error('AccessDenied');
+                }
+                return JSON.stringify(listing);
+            }
+            return run(command, args);
+        };
+        if (
+            listing.KeyCount === 0 &&
+            listing.IsTruncated === false &&
+            !('Contents' in listing)
+        ) {
+            await publishMacos(f);
+            assert.equal(f.objects.size, 8);
+        } else {
+            await assert.rejects(publishMacos(f), /NoneType|AccessDenied/);
+            assert.equal(f.objects.get(key).toString(), 'unfinished old build');
+            assert.equal(f.objects.size, 1);
+        }
+    });
+}
+
 for (const status of [403, 404, 408, 429, 500, 502, 503, 504, 'network']) {
     test(`temporary public download failure ${status} recovers before completion`, async (t) => {
         const f = fixture(t);
@@ -431,13 +645,13 @@ test('rejects a corrupt updater archive before publishing any object', async (t)
     assert.equal(f.objects.size, 0);
 });
 
-test('atomically promotes a newer feed only after all immutable objects are public', async (t) => {
+test('atomically promotes a newer feed only after all release files are public', async (t) => {
     const f = fixture(t);
     const latestKey = `releases/macos/${f.metadata.target}/latest.json`;
     f.objects.set(latestKey, Buffer.from(JSON.stringify({ version: '1.0.0' })));
     await publishMacos(f);
     assert.equal(JSON.parse(f.objects.get(latestKey)).version, '1.2.3');
-    const immutableVerified = f.events.findIndex(
+    const releaseVerified = f.events.findIndex(
         (event) =>
             event.operation === 'public-get' &&
             event.key.endsWith('/update.json'),
@@ -445,7 +659,7 @@ test('atomically promotes a newer feed only after all immutable objects are publ
     const latestPut = f.events.findIndex(
         (event) => event.operation === 'put-object' && event.key === latestKey,
     );
-    assert(latestPut > immutableVerified);
+    assert(latestPut > releaseVerified);
 });
 
 for (const previous of [
@@ -496,12 +710,16 @@ for (const message of [
         assert.deepEqual([...f.objects.entries()], published);
     });
 
-    test(`empty S3 error cannot bypass immutable content checks: ${message}`, async (t) => {
+    test(`empty S3 error cannot replace a published release: ${message}`, async (t) => {
         const f = fixture(t, message);
         const key = `releases/macos/${f.metadata.target}/1.2.3/${f.metadata.filename}`;
         f.objects.set(key, Buffer.from('different existing release'));
+        f.objects.set(
+            `releases/macos/${f.metadata.target}/latest.json`,
+            Buffer.from(JSON.stringify({ version: '1.2.3' })),
+        );
         await assert.rejects(publishMacos(f), /bump the app version/);
-        assert.equal(f.objects.size, 1);
+        assert.equal(f.objects.size, 2);
         assert.equal(
             f.objects.get(key).toString(),
             'different existing release',