diff --git a/apps/obsidian/LICENSE b/apps/obsidian/LICENSE new file mode 100644 index 000000000..e73a4024e --- /dev/null +++ b/apps/obsidian/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [2024] [Homeworld Collective] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/apps/obsidian/scripts/publish.ts b/apps/obsidian/scripts/publish.ts index 781e29e7c..c60bbad8b 100644 --- a/apps/obsidian/scripts/publish.ts +++ b/apps/obsidian/scripts/publish.ts @@ -1,8 +1,10 @@ +/* eslint-disable */ import dotenv from "dotenv"; import fs from "fs"; import path from "path"; import { exec } from "child_process"; import util from "util"; +import crypto from "crypto"; // https://linear.app/discourse-graphs/issue/ENG-766/upgrade-all-commonjs-to-esm // TODO if possible: change apps/obsidian to ESM. Use require until then. // import { Octokit } from "@octokit/core"; @@ -35,6 +37,7 @@ const EXCLUDE_PATTERNS = [ ".next", "out", "build", + "scripts", ".git", ".vscode", ".cursor", @@ -47,6 +50,9 @@ const REQUIRED_BUILD_FILES = [ "manifest.json", "styles.css", ] as const; +const BLOB_UPLOAD_BATCH_SIZE = 10; +const MAX_GITHUB_RETRIES = 5; +const BASE_RETRY_DELAY_MS = 2_000; const TARGET_REPO = "DiscourseGraphs/discourse-graph-obsidian"; const OWNER = "DiscourseGraphs"; @@ -56,6 +62,96 @@ const log = (message: string): void => { console.log(`[Obsidian Publisher] ${message}`); }; +const sleep = async (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +const isSecondaryRateLimitError = (error: unknown): boolean => { + const maybeError = error as { + status?: number; + response?: { data?: { message?: string } }; + message?: string; + }; + const message = + maybeError?.response?.data?.message?.toLowerCase() ?? + maybeError?.message?.toLowerCase() ?? + ""; + return maybeError?.status === 403 && message.includes("secondary rate limit"); +}; + +const getRetryDelayMs = (error: unknown, attempt: number): number => { + const maybeError = error as { + response?: { headers?: Record }; + }; + const retryAfterHeader = maybeError?.response?.headers?.["retry-after"]; + const retryAfterSeconds = Number(retryAfterHeader); + if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) { + return retryAfterSeconds * 1000; + } + return BASE_RETRY_DELAY_MS * 2 ** attempt; +}; + +const requestWithRetry = async ( + request: () => Promise, + context: string, +): Promise => { + let attempt = 0; + + while (true) { + try { + return await request(); + } catch (error) { + if (!isSecondaryRateLimitError(error) || attempt >= MAX_GITHUB_RETRIES) { + throw error; + } + + const delayMs = getRetryDelayMs(error, attempt); + log( + `Secondary rate limit hit during ${context}. Retrying in ${Math.ceil(delayMs / 1000)}s (attempt ${attempt + 1}/${MAX_GITHUB_RETRIES})...`, + ); + await sleep(delayMs); + attempt += 1; + } + } +}; + +const getAllFiles = (dir: string, baseDir: string = dir): string[] => { + const files: string[] = []; + + fs.readdirSync(dir, { withFileTypes: true }).forEach((entry) => { + const fullPath = path.join(dir, entry.name); + const relativePath = path.relative(baseDir, fullPath); + + if (shouldExclude(fullPath, baseDir)) { + log(`Excluding: ${relativePath}`); + return; + } + + if (entry.isDirectory()) { + files.push(...getAllFiles(fullPath, baseDir)); + } else { + files.push(relativePath); + } + }); + + return files; +}; + +const getGitBlobSha = (content: Buffer): string => { + const header = Buffer.from(`blob ${content.length}\0`, "utf8"); + return crypto + .createHash("sha1") + .update(Buffer.concat([header, content])) + .digest("hex"); +}; + +const chunk = (items: T[], size: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < items.length; i += size) { + chunks.push(items.slice(i, i + size)); + } + return chunks; +}; + const getEnvVar = (name: string): string => { const value = process.env[name]; if (!value) { @@ -294,6 +390,18 @@ const copyBuildFiles = (buildDir: string, tempDir: string): void => { }); }; +const sanitizePackageJsonForMirror = (tempDir: string): void => { + const packageJsonPath = path.join(tempDir, "package.json"); + if (!fs.existsSync(packageJsonPath)) return; + + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); + if (packageJson?.scripts) { + delete packageJson.scripts; + fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2)); + log("Removed package.json scripts for mirrored publish repo"); + } +}; + const updateMainBranch = async ( tempDir: string, version: string, @@ -306,13 +414,14 @@ const updateMainBranch = async ( const repo = REPO; try { - const { data: ref } = await octokit.request( - "GET /repos/{owner}/{repo}/git/refs/{ref}", - { - owner, - repo, - ref: "heads/main", - }, + const { data: ref } = await requestWithRetry( + () => + octokit.request("GET /repos/{owner}/{repo}/git/refs/{ref}", { + owner, + repo, + ref: "heads/main", + }), + "fetching main branch ref", ); if (!ref?.object?.sha) { @@ -320,114 +429,165 @@ const updateMainBranch = async ( } const currentSha = ref.object.sha; - const { data: currentCommit } = await octokit.request( - "GET /repos/{owner}/{repo}/git/commits/{commit_sha}", - { - owner, - repo, - commit_sha: currentSha, - }, + const { data: currentCommit } = await requestWithRetry( + () => + octokit.request("GET /repos/{owner}/{repo}/git/commits/{commit_sha}", { + owner, + repo, + commit_sha: currentSha, + }), + "fetching current main commit", ); if (!currentCommit?.tree?.sha) { throw new Error("Failed to get current commit tree"); } const currentTreeSha = currentCommit.tree.sha; + const { data: existingTree } = await requestWithRetry( + () => + octokit.request("GET /repos/{owner}/{repo}/git/trees/{tree_sha}", { + owner, + repo, + tree_sha: currentTreeSha, + recursive: "1", + }), + "fetching recursive main tree", + ); - const getAllFiles = (dir: string, baseDir: string = dir): string[] => { - const files: string[] = []; - - fs.readdirSync(dir, { withFileTypes: true }).forEach((entry) => { - const fullPath = path.join(dir, entry.name); - const relativePath = path.relative(baseDir, fullPath); - - if (shouldExclude(fullPath, baseDir)) { - log(`Excluding: ${relativePath}`); - return; - } - - if (entry.isDirectory()) { - files.push(...getAllFiles(fullPath, baseDir)); - } else { - files.push(relativePath); - } - }); - - return files; - }; + const existingBlobShasByPath = new Map( + (existingTree.tree ?? []) + .filter( + (entry: any): entry is { path: string; sha: string; type: string } => + Boolean(entry.path && entry.sha && entry.type === "blob"), + ) + .map((entry: { path: string; sha: string }) => [entry.path, entry.sha]), + ); const allFiles = getAllFiles(tempDir); log(`Found ${allFiles.length} files to update`); - - const blobPromises = allFiles.map(async (filePath) => { + const normalizedAllFiles = allFiles.map((filePath) => + filePath.replace(/\\/g, "/"), + ); + const currentRepoFiles = new Set(existingBlobShasByPath.keys()); + const localFiles = new Set(normalizedAllFiles); + const filesToDelete = [...currentRepoFiles].filter( + (repoFilePath) => !localFiles.has(repoFilePath), + ); + const filesToUpdate = allFiles.filter((filePath) => { const fullPath = path.join(tempDir, filePath); const content = fs.readFileSync(fullPath); + const normalizedPath = filePath.replace(/\\/g, "/"); + const existingSha = existingBlobShasByPath.get(normalizedPath); + return getGitBlobSha(content) !== existingSha; + }); - const { data: blob } = await octokit.request( - "POST /repos/{owner}/{repo}/git/blobs", - { - owner, - repo, - content: content.toString("base64"), - encoding: "base64", - }, + log( + `Detected ${filesToUpdate.length} changed files (${allFiles.length - filesToUpdate.length} unchanged skipped)`, + ); + log(`Detected ${filesToDelete.length} files to delete from target repo`); + + if (filesToUpdate.length === 0 && filesToDelete.length === 0) { + log("No changes detected on main branch; skipping commit update"); + return; + } + + const blobBatchChunks = chunk(filesToUpdate, BLOB_UPLOAD_BATCH_SIZE); + const blobs: Array<{ path: string; sha: string }> = []; + + for (const [batchIndex, blobBatch] of blobBatchChunks.entries()) { + log( + `Uploading blob batch ${batchIndex + 1}/${blobBatchChunks.length} (${blobBatch.length} files)...`, ); - if (!blob?.sha) { - throw new Error(`Failed to create blob for ${filePath}`); - } + const batchBlobs = await Promise.all( + blobBatch.map(async (filePath) => { + const fullPath = path.join(tempDir, filePath); + const content = fs.readFileSync(fullPath); + + const { data: blob } = await requestWithRetry( + () => + octokit.request("POST /repos/{owner}/{repo}/git/blobs", { + owner, + repo, + content: content.toString("base64"), + encoding: "base64", + }), + `creating blob for ${filePath}`, + ); - return { - path: filePath.replace(/\\/g, "/"), // Normalize path separators for GitHub - sha: blob.sha, - }; - }); + if (!blob?.sha) { + throw new Error(`Failed to create blob for ${filePath}`); + } - const blobs = await Promise.all(blobPromises); - - const { data: newTree } = await octokit.request( - "POST /repos/{owner}/{repo}/git/trees", - { - owner, - repo, - base_tree: currentTreeSha, - tree: blobs.map((blob) => ({ - path: blob.path, - mode: "100644" as const, - type: "blob" as const, - sha: blob.sha, - })), - }, + return { + path: filePath.replace(/\\/g, "/"), // Normalize path separators for GitHub + sha: blob.sha, + }; + }), + ); + + blobs.push(...batchBlobs); + } + + const treeUpdates = blobs.map((blob) => ({ + path: blob.path, + mode: "100644" as const, + type: "blob" as const, + sha: blob.sha, + })); + const treeDeletions = filesToDelete.map((filePath) => ({ + path: filePath, + mode: "100644" as const, + type: "blob" as const, + sha: null, + })); + + const { data: newTree } = await requestWithRetry( + () => + octokit.request("POST /repos/{owner}/{repo}/git/trees", { + owner, + repo, + base_tree: currentTreeSha, + tree: [...treeUpdates, ...treeDeletions], + }), + "creating updated git tree", ); if (!newTree?.sha) { throw new Error("Failed to create new tree"); } - const { data: newCommit } = await octokit.request( - "POST /repos/{owner}/{repo}/git/commits", - { - owner, - repo, - message: `Release v${version}`, - tree: newTree.sha, - parents: [currentSha], - }, + const { data: newCommit } = await requestWithRetry( + () => + octokit.request("POST /repos/{owner}/{repo}/git/commits", { + owner, + repo, + message: `Release v${version}`, + tree: newTree.sha, + parents: [currentSha], + }), + "creating release commit", ); if (!newCommit?.sha) { throw new Error("Failed to create new commit"); } - await octokit.request("PATCH /repos/{owner}/{repo}/git/refs/{ref}", { - owner, - repo, - ref: "heads/main", - sha: newCommit.sha, - }); + await requestWithRetry( + () => + octokit.request("PATCH /repos/{owner}/{repo}/git/refs/{ref}", { + owner, + repo, + ref: "heads/main", + sha: newCommit.sha, + }), + "updating main branch reference", + ); log(`Successfully updated main branch with commit: ${newCommit.sha}`); - log(`Updated ${blobs.length} files`); + log( + `Updated ${blobs.length} files and deleted ${filesToDelete.length} files`, + ); } catch (error) { log(`Failed to update main branch: ${error}`); throw error; @@ -447,7 +607,7 @@ const createGithubRelease = async ({ const octokit = new Octokit({ auth: token }); const owner = OWNER; const repo = REPO; - const tagName = `v${version}`; + const tagName = `${version}`; const releaseTitle = releaseName || `Discourse Graph v${version}`; const isPrerelease = !isExternalRelease(version); @@ -469,16 +629,17 @@ const createGithubRelease = async ({ fs.copyFileSync(manifestSrc, manifestDest); updateManifest(releaseTempDir, version); - const release = await octokit.request( - "POST /repos/{owner}/{repo}/releases", - { - owner, - repo, - tag_name: tagName, - name: releaseTitle, - prerelease: isPrerelease, - generate_release_notes: true, - }, + const release = await requestWithRetry( + () => + octokit.request("POST /repos/{owner}/{repo}/releases", { + owner, + repo, + tag_name: tagName, + name: releaseTitle, + prerelease: isPrerelease, + generate_release_notes: true, + }), + "creating GitHub release", ); if (!release.data.upload_url) { @@ -503,14 +664,18 @@ const createGithubRelease = async ({ `?name=${file}`, ); - await octokit.request(`POST ${uploadUrl}`, { - headers: { - "content-type": contentType, - "content-length": String(stats.size), - }, - data: fileContent, - name: file, - }); + await requestWithRetry( + () => + octokit.request(`POST ${uploadUrl}`, { + headers: { + "content-type": contentType, + "content-length": String(stats.size), + }, + data: fileContent, + name: file, + }), + `uploading release asset ${file}`, + ); log(`Uploaded ${file}`); } @@ -540,6 +705,7 @@ const publish = async (config: PublishConfig): Promise => { copyDirectory({ src: obsidianDir, dest: tempDir, baseDir: obsidianDir }); copyBuildFiles(buildDir, tempDir); + sanitizePackageJsonForMirror(tempDir); if (isExternal) { updateManifest(tempDir, version);