logger = $logger; $this->name = $name; $this->filter = $filter; $this->exclusions = $exclusions; } /** @inheritDoc * @throws Exception */ public function update(EGGDatabase $db) { $this->preUpdate(); $repos = $this->listAndUpdateRepositories($db); $anyChanged = false; foreach ($repos as $repo) { $db->beginTransaction(); { $branches = $this->listAndUpdateBranches($db, $repo); $db->setUpdateDateOnRepository($repo); $repo_changed = false; foreach ($branches as $branch) { if ($branch->HeadFromAPI === $branch->Head) { $db->setUpdateDateOnBranch($branch); $this->logger->proclog("Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] is up to date"); continue; } $updateCount = $this->listAndUpdateCommits($db, $repo, $branch); $db->setUpdateDateOnBranch($branch); if ($updateCount === 0) { $this->logger->proclog("Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] has no new commits"); continue; } $this->logger->proclog("Found " . $updateCount . " new commits in Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "]"); $repo_changed = true; $db->setChangeDateOnBranch($branch); } if ($repo_changed) $db->setChangeDateOnRepository($repo); if ($repo_changed) $anyChanged = true; } $db->commitTransaction(); } if ($anyChanged) { $this->logger->proclog("Deleting dangling commit-data..."); $db->deleteDanglingCommitdata(); } $this->postUpdate(); $this->logger->proclog("Finished [" . $this->name . "]"); } /** * @throws Exception */ protected abstract function preUpdate(); /** * @throws Exception */ protected abstract function postUpdate(); /** * @param int $page * @return array */ protected abstract function queryOrganizations($page); /** * @param string $user * @param int $page * @return array */ protected abstract function queryRepositories($user, $page); /** * @param string $reponame * @param int $page * @return array */ protected abstract function queryBranches($reponame, $page); /** * @param string $reponame * @param string $branchname * @param string $startsha * @return array */ protected abstract function queryCommits($reponame, $branchname, $startsha); /** * @param mixed $data * @return array * @throws Exception */ protected abstract function readOrganization($data); /** * @param mixed $data * @return array * @throws Exception */ protected abstract function readRepository($data); /** * @param mixed $data * @return array * @throws Exception */ protected abstract function readBranch($data); /** * @param mixed $data * @return array * @throws Exception */ protected abstract function readCommit($data); /** * @param EGGDatabase $db * @return Repository[] * @throws Exception */ private function listAndUpdateRepositories(EGGDatabase $db) { $f = explode('/', $this->filter); $result = []; if ($f[0] === '*') { $page = 1; $json = $this->queryOrganizations($page); while (! empty($json)) { foreach ($json as $result_org) { $jdata = $this->readOrganization($result_org); $this->logger->proclog("Found Org in Remote: " . $jdata['full_name'] . " (" . $jdata['key'] . ")"); $r0 = $this->listAndUpdateRepositoriesOfOrganization($db, $jdata['key']); $result = array_merge($result, $r0); } $page++; $json = $this->queryOrganizations($page); } } else { $r0 = $this->listAndUpdateRepositoriesOfOrganization($db, $f[0]); $result = array_merge($result, $r0); } $db->deleteOtherRepositories($this->name, $result); return $result; } private function listAndUpdateRepositoriesOfOrganization(EGGDatabase $db, string $org){ $result = []; $page = 1; $json = $this->queryRepositories($org, $page); $found = []; while (! empty($json)) { $count = 0; foreach ($json as $result_repo) { $jdata = $this->readRepository($result_repo); if (in_array($jdata['full_name'], $found)) continue; $found []= $jdata['full_name']; if (!Utils::isRepoFilterMatch($this->filter, $this->exclusions, $jdata['full_name'])) { $this->logger->proclog("Skip Repo: " . $jdata['full_name']); continue; } $this->logger->proclog("Found Repo in Remote: " . $jdata['full_name']); $result []= $db->getOrCreateRepository($jdata['html_url'], $jdata['full_name'], $this->name); $count++; } if ($count === 0) break; $page++; $json = $this->queryRepositories($org, $page); } return $result; } /** * @param EGGDatabase $db * @param Repository $repo * @return Branch[] * @throws Exception */ private function listAndUpdateBranches(EGGDatabase $db, Repository $repo) { $result = []; $page = 1; $json = $this->queryBranches($repo->Name, $page); $found = []; while (! empty($json)) { $count = 0; foreach ($json as $result_branch) { $jdata = $this->readBranch($result_branch); if ($jdata === null) continue; $bname = $jdata['name']; $bhead = $jdata['sha']; if (in_array($bname, $found)) continue; $found []= $bname; $this->logger->proclog("Found Branch in Remote: [" . $repo->Name . "] " . $bname); $b = $db->getOrCreateBranch($this->name, $repo, $bname); $b->HeadFromAPI = $bhead; $result []= $b; $count++; } if ($count === 0) break; $page++; $json = $this->queryBranches($repo->Name, $page); } $db->deleteOtherBranches($this->name, $repo, $result); return $result; } /** * @param EGGDatabase $db * @param Repository $repo * @param Branch $branch * @return Commit[] * @throws Exception */ private function listAndUpdateCommits(EGGDatabase $db, Repository $repo, Branch $branch): int { if ($branch->Head !== null && $branch->HeadFromAPI === $branch->Head) return 0; // nothing to do /** @var Commit[] $queried_commits */ $queried_commits = []; if ($branch->HeadFromAPI === null) return []; $target = $branch->Head; $oldHeadFound = false; $next_sha = [ $branch->HeadFromAPI ]; $queryCounter=0; $reusedFromExistingCommitData = 0; $queriedFromAPI = 0; $visited = []; /** @var Commit[] $existingCommitData (hash -> Commit) */ $existingCommitData = []; $this->logger->proclog("Query existing commits for [" . $this->name . "|" . $repo->Name . "] (re-use)"); foreach ($db->getCommitdataForRepo($repo, $branch) as $c) $existingCommitData[$c->Hash] = $c; $this->logger->proclog("Found " . count($existingCommitData) . " existing commit-data in DB"); /** @var Commit[] $unprocessed */ $unprocessed = []; for (;;) { foreach ($unprocessed as $commit) { while (($rmshakey = array_search($commit->Hash, $next_sha)) !== false) unset($next_sha[$rmshakey]); if (in_array($commit->Hash, $visited)) continue; $visited []= $commit->Hash; if ($commit->Hash === $target) $oldHeadFound = true; $queried_commits []= $commit; foreach ($commit->Parents as $p) { $next_sha []= $p; } } $next_sha = array_values($next_sha); // fix numeric keys if (count($next_sha) === 0) break; // all leafs were processed, teh whole graph shoul have been handled if (array_key_exists($next_sha[0], $existingCommitData)) { // fast-track for existing Commit-Data $unprocessed = [ $existingCommitData[$next_sha[0]] ]; $reusedFromExistingCommitData++; } else { $queryCounter++; if ($queryCounter === 1) { $this->logger->proclog("Query commits for [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] (initial @ {" . substr($next_sha[0], 0, 8) . "}) (target: {" . substr($target ?? 'NULL', 0, 8) . "})"); } else { $this->logger->proclog("Query commits for [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] (" . $queryCounter . " @ {" . substr($next_sha[0], 0, 8) . "})"); } $unprocessed = array_map(fn($p) => $this->createCommit($branch, $p), $this->queryCommits($repo->Name, $branch->Name, $next_sha[0])); $queriedFromAPI += count($unprocessed); } } if ($branch->Head === null) { // farm-fresh new branch $this->logger->proclog("HEAD pointer in new Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] set to {".substr($branch->HeadFromAPI ?? 'NULL', 0, 8)."} - Queried " . count($queried_commits) . " commits (reused $reusedFromExistingCommitData commits from DB)"); $db->insertNewCommits($this->name, $repo, $branch, $queried_commits); $db->setBranchHead($branch, $branch->HeadFromAPI); return count($queried_commits); } else if ($oldHeadFound) { // normal update, a few commits were added $this->logger->proclog("Query existing commits in DB for [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "]"); $commitsInDB = array_map(function(Commit $m):string{return $m->Hash;}, $db->getCommitsForBranch($branch)); $actual_new_commits = array_filter($queried_commits, fn($p) => !in_array($p->Hash, $commitsInDB)); $this->logger->proclog("Update Branch [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] HEAD from {".substr($branch->Head ?? 'NULL', 0, 8)."} to {".substr($branch->HeadFromAPI ?? 'NULL', 0, 8)."} by adding ".count($actual_new_commits)." new commits (total = ".count($queried_commits).")"); $db->insertNewCommits($this->name, $repo, $branch, $actual_new_commits); $db->setBranchHead($branch, $branch->HeadFromAPI); return count($actual_new_commits); } else { // the old head was no longer found, some commits need to be deleted, do a full re-sync with the DB $this->logger->proclog("HEAD pointer in Branch: [" . $this->name . "|" . $repo->Name . "|" . $branch->Name . "] no longer matches. Fully updating all " . count($queried_commits) . " commits (changing HEAD from {".substr($branch->Head ?? 'NULL', 0, 8)."} to {".substr($branch->HeadFromAPI ?? 'NULL', 0, 8)."} (reused $reusedFromExistingCommitData commits from DB)"); $db->deleteAllCommits($branch); $db->insertNewCommits($this->name, $repo, $branch, $queried_commits); $db->setBranchHead($branch, $branch->HeadFromAPI); return count($queried_commits); } } private function createCommit(Branch $branch, $result_commit): Commit { $jdata = $this->readCommit($result_commit); $sha = $jdata['sha']; $author_name = $jdata['author_name']; $author_email = $jdata['author_email']; $committer_name = $jdata['committer_name']; $committer_email = $jdata['committer_email']; $message = $jdata['message']; $date = $jdata['date']; $parents = $jdata['parents']; $commit = new Commit(); $commit->Branch = $branch; $commit->Hash = $sha; $commit->AuthorName = $author_name; $commit->AuthorEmail = $author_email; $commit->CommitterName = $committer_name; $commit->CommitterEmail = $committer_email; $commit->Message = $message; $commit->Date = $date; $commit->Parents = $parents; return $commit; } /** @inheritDoc */ public function getName() { return $this->name; } /** @inheritDoc */ public abstract function toString(); }