diff --git a/www/extern/egg/ExtendedGitGraph2.php b/www/extern/egg/ExtendedGitGraph2.php index 065d8b9..a319f12 100644 --- a/www/extern/egg/ExtendedGitGraph2.php +++ b/www/extern/egg/ExtendedGitGraph2.php @@ -3,6 +3,8 @@ require_once 'Logger.php'; require_once 'EGGException.php'; require_once 'RemoteSource.php'; +require_once 'RemoteSourceGithub.php'; +require_once 'RemoteSourceGitea.php'; require_once 'OutputGenerator.php'; require_once 'EGGDatabase.php'; require_once 'Utils.php'; diff --git a/www/extern/egg/RemoteSource.php b/www/extern/egg/RemoteSource.php index 04324f5..62fc0a0 100644 --- a/www/extern/egg/RemoteSource.php +++ b/www/extern/egg/RemoteSource.php @@ -115,6 +115,12 @@ abstract class StandardGitConnection implements IRemoteSource */ protected abstract function postUpdate(); + /** + * @param int $page + * @return array + */ + protected abstract function queryOrganizations($page); + /** * @param string $user * @param int $page @@ -137,6 +143,13 @@ abstract class StandardGitConnection implements IRemoteSource */ protected abstract function queryCommits($reponame, $branchname, $startsha); + /** + * @param mixed $data + * @return array + * @throws Exception + */ + protected abstract function readOrganization($data); + /** * @param mixed $data * @return array @@ -166,45 +179,81 @@ abstract class StandardGitConnection implements IRemoteSource private function listAndUpdateRepositories(EGGDatabase $db) { $f = explode('/', $this->filter); - $result = []; + $result = []; - $page = 1; - $json = $this->queryRepositories($f[0], $page); + if ($f[0] === '*') { - $found = []; - while (! empty($json)) - { - $count = 0; - foreach ($json as $result_repo) - { - $jdata = $this->readRepository($result_repo); + $page = 1; + $json = $this->queryOrganizations($page); + while (! empty($json)) { - if (in_array($jdata['full_name'], $found)) continue; - $found []= $jdata['full_name']; + foreach ($json as $result_org) { - if (!Utils::isRepoFilterMatch($this->filter, $this->exclusions, $jdata['full_name'])) - { - $this->logger->proclog("Skip Repo: " . $jdata['full_name']); - continue; - } + $jdata = $this->readOrganization($result_org); - $this->logger->proclog("Found Repo in Remote: " . $jdata['full_name']); + $this->logger->proclog("Found Org in Remote: " . $jdata['full_name'] . " (" . $jdata['key'] . ")"); - $result []= $db->getOrCreateRepository($jdata['html_url'], $jdata['full_name'], $this->name); - $count++; - } + $r0 = $this->listAndUpdateRepositoriesOfOrganization($db, $jdata['key']); + $result = array_merge($result, $r0); - if ($count === 0) break; + } - $page++; - $json = $this->queryRepositories($f[0], $page); - } + $page++; + $json = $this->queryOrganizations($page); + } - $db->deleteOtherRepositories($this->name, $result); + + } else { + + $r0 = $this->listAndUpdateRepositoriesOfOrganization($db, $f[0]); + $result = array_merge($result, $r0); + + } + + $db->deleteOtherRepositories($this->name, $result); return $result; } + private function listAndUpdateRepositoriesOfOrganization(EGGDatabase $db, string $org){ + + $result = []; + + $page = 1; + $json = $this->queryRepositories($org, $page); + + $found = []; + while (! empty($json)) + { + $count = 0; + foreach ($json as $result_repo) + { + $jdata = $this->readRepository($result_repo); + + if (in_array($jdata['full_name'], $found)) continue; + $found []= $jdata['full_name']; + + if (!Utils::isRepoFilterMatch($this->filter, $this->exclusions, $jdata['full_name'])) + { + $this->logger->proclog("Skip Repo: " . $jdata['full_name']); + continue; + } + + $this->logger->proclog("Found Repo in Remote: " . $jdata['full_name']); + + $result []= $db->getOrCreateRepository($jdata['html_url'], $jdata['full_name'], $this->name); + $count++; + } + + if ($count === 0) break; + + $page++; + $json = $this->queryRepositories($org, $page); + } + + return $result; + } + /** * @param EGGDatabase $db * @param Repository $repo @@ -412,264 +461,3 @@ abstract class StandardGitConnection implements IRemoteSource /** @inheritDoc */ public abstract function toString(); } - -class GithubConnection extends StandardGitConnection -{ - const API_OAUTH_AUTH = 'https://github.com/login/oauth/authorize?client_id=%s'; - const URL_OAUTH_TOKEN = 'https://github.com/login/oauth/access_token?client_id={id}&client_secret={secret}&code={code}'; - - const API_RATELIMIT = 'https://api.github.com/rate_limit'; - const API_REPOSITORIESLIST = 'https://api.github.com/users/{user}/repos?page={page}&per_page=100'; - const API_COMMITSLIST = 'https://api.github.com/repos/{repo}/commits?per_page=100&sha={sha}'; - const API_BRANCHLIST = 'https://api.github.com/repos/{repo}/branches?page={page}'; - - /** @var string $url */ - private $url; - - /** @var string $oauth_id */ - private $oauth_id; - - /** @var string $oauth_secret */ - private $oauth_secret; - - /** @var string $apitokenpath */ - private $apitokenpath; - - /** @var string $apitoken */ - private $apitoken; - - /** - * @param ILogger $logger - * @param string $name - * @param string $url - * @param string $filter - * @param string[] exclusions - * @param string $oauth_id - * @param string $oauth_secret - * @param string $apitokenpath - */ - public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $oauth_id, string $oauth_secret, string $apitokenpath) - { - parent::__construct($logger, $name, $filter, $exclusions); - - $this->url = $url; - $this->oauth_id = $oauth_id; - $this->oauth_secret = $oauth_secret; - $this->apitokenpath = $apitokenpath; - - if ($this->apitokenpath !== null && file_exists($this->apitokenpath)) - $this->apitoken = file_get_contents($this->apitokenpath); - else - $this->apitoken = null; - } - - /** - * @throws Exception - */ - public function queryAPIToken() { - $url = Utils::sharpFormat(self::URL_OAUTH_TOKEN, ['id'=>$this->oauth_id, 'secret'=>$this->oauth_secret, 'code'=>'egg']); - $fullresult = $result = file_get_contents($url); - - if (Utils::startsWith($fullresult, 'error=')) throw new EGGException('GitHub Auth failed: ' . $fullresult); - - $result = str_replace('access_token=', '', $result); - $result = str_replace('&scope=&token_type=bearer', '', $result); - - $this->logger->proclog("Updated Github API token"); - - if ($result!=='' && $result !== null && $this->apitokenpath !== null) - file_put_contents($this->apitokenpath, $result); - - $this->apitoken = $result; - } - - /** @inheritDoc */ - protected function preUpdate() - { - if ($this->apitoken === null) $this->queryAPIToken(); - } - - /** @inheritDoc */ - protected function postUpdate() - { - // - } - - /** @inheritDoc */ - protected function queryRepositories($user, $page) - { - $url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page]); - return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); - } - - /** @inheritDoc */ - protected function queryBranches($reponame, $page) - { - $url = Utils::sharpFormat(self::API_BRANCHLIST, ['repo'=>$reponame, 'page'=>$page]); - return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); - } - - /** @inheritDoc */ - protected function queryCommits($reponame, $branchname, $startsha) - { - $url = Utils::sharpFormat(self::API_COMMITSLIST, [ 'repo'=>$reponame, 'sha'=>$startsha ]); - return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); - } - - /** @inheritDoc */ - protected function readRepository($data) - { - return - [ - 'full_name' => $data->{'full_name'}, - 'html_url' => $data->{'html_url'}, - ]; - } - - /** @inheritDoc */ - protected function readBranch($data) - { - if (isset($data->{'block'})) return null; - - return - [ - 'name' => $data->{'name'}, - 'sha' => $data->{'commit'}->{'sha'}, - ]; - } - - /** @inheritDoc */ - protected function readCommit($data) - { - return - [ - 'sha' => $data->{'sha'}, - - 'author_name' => $data->{'commit'}->{'author'}->{'name'}, - 'author_email' => $data->{'commit'}->{'author'}->{'email'}, - - 'committer_name' => $data->{'commit'}->{'committer'}->{'name'}, - 'committer_email' => $data->{'commit'}->{'committer'}->{'email'}, - - 'message' => $data->{'commit'}->{'message'}, - 'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"), - - 'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}), - ]; - } - - /** @inheritDoc */ - public function toString() { return "[Github|".$this->filter."]"; } -} - -class GiteaConnection extends StandardGitConnection -{ - const API_BASE_URL = '/api/v1'; - - const API_USER_REPO_LIST = '/users/{user}/repos?page={page}&limit={limit}'; - const API_BRANCH_LIST = '/repos/{repo}/branches?page={page}&limit={limit}'; - const API_COMMIT_LIST = '/repos/{repo}/commits?sha={sha}&limit={limit}'; - - /** @var string $url */ - private $url; - - /** @var string $username */ - private $username; - - /** @var string $password */ - private $password; - - /** - * @param ILogger $logger - * @param string $name - * @param string $url - * @param string $filter - * @param string[] $exclusions - * @param string $username - * @param string $password - */ - public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $username, string $password) - { - parent::__construct($logger, $name, $filter, $exclusions); - - $this->url = $url; - $this->username = $username; - $this->password = $password; - } - - /** @inheritDoc */ - protected function preUpdate() - { - // - } - - /** @inheritDoc */ - protected function postUpdate() - { - // - } - - /** @inheritDoc */ - protected function queryRepositories($user, $page) - { - $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_REPO_LIST), ['user'=>$user, 'page'=>$page, 'limit'=>64 ]); - return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); - } - - /** @inheritDoc */ - protected function queryBranches($reponame, $page) - { - $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_BRANCH_LIST), ['repo'=>$reponame, 'page'=>$page, 'limit'=>64]); - return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); - } - - /** @inheritDoc */ - protected function queryCommits($reponame, $branchname, $startsha) - { - $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_COMMIT_LIST), [ 'repo'=>$reponame, 'sha'=>$startsha, 'limit'=>1024 ]); - return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); - } - - /** @inheritDoc */ - protected function readRepository($data) - { - return - [ - 'full_name' => $data->{'full_name'}, - 'html_url' => $data->{'html_url'}, - ]; - } - - /** @inheritDoc */ - protected function readBranch($data) - { - return - [ - 'name' => $data->{'name'}, - 'sha' => $data->{'commit'}->{'id'}, - ]; - } - - /** @inheritDoc */ - protected function readCommit($data) - { - return - [ - 'sha' => $data->{'commit'}->{'tree'}->{'sha'}, - - 'author_name' => $data->{'commit'}->{'author'}->{'name'}, - 'author_email' => $data->{'commit'}->{'author'}->{'email'}, - - 'committer_name' => $data->{'commit'}->{'committer'}->{'name'}, - 'committer_email' => $data->{'commit'}->{'committer'}->{'email'}, - - 'message' => $data->{'commit'}->{'message'}, - 'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"), - - 'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}), - ]; - } - - /** @inheritDoc */ - public function toString() { return "[Gitea|".$this->url."|".$this->filter."]"; } -} \ No newline at end of file diff --git a/www/extern/egg/RemoteSourceGitea.php b/www/extern/egg/RemoteSourceGitea.php new file mode 100644 index 0000000..4e89b0e --- /dev/null +++ b/www/extern/egg/RemoteSourceGitea.php @@ -0,0 +1,136 @@ +url = $url; + $this->username = $username; + $this->password = $password; + } + + /** @inheritDoc */ + protected function preUpdate() + { + // + } + + /** @inheritDoc */ + protected function postUpdate() + { + // + } + + /** @inheritDoc */ + protected function queryOrganizations($page) + { + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_ORG_LIST), ['page'=>$page, 'limit'=>64 ]); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function queryRepositories($user, $page) + { + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_REPO_LIST), ['user'=>$user, 'page'=>$page, 'limit'=>64 ]); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function queryBranches($reponame, $page) + { + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_BRANCH_LIST), ['repo'=>$reponame, 'page'=>$page, 'limit'=>64]); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function queryCommits($reponame, $branchname, $startsha) + { + $url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_COMMIT_LIST), [ 'repo'=>$reponame, 'sha'=>$startsha, 'limit'=>1024 ]); + return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password); + } + + /** @inheritDoc */ + protected function readRepository($data) + { + return + [ + 'full_name' => $data->{'full_name'}, + 'html_url' => $data->{'html_url'}, + ]; + } + + /** @inheritDoc */ + protected function readOrganization($data) + { + return + [ + 'id' => $data->{'id'}, + 'full_name' => $data->{'full_name'}, + 'key' => $data->{'username'}, + ]; + } + + /** @inheritDoc */ + protected function readBranch($data) + { + return + [ + 'name' => $data->{'name'}, + 'sha' => $data->{'commit'}->{'id'}, + ]; + } + + /** @inheritDoc */ + protected function readCommit($data) + { + return + [ + 'sha' => $data->{'commit'}->{'tree'}->{'sha'}, + + 'author_name' => $data->{'commit'}->{'author'}->{'name'}, + 'author_email' => $data->{'commit'}->{'author'}->{'email'}, + + 'committer_name' => $data->{'commit'}->{'committer'}->{'name'}, + 'committer_email' => $data->{'commit'}->{'committer'}->{'email'}, + + 'message' => $data->{'commit'}->{'message'}, + 'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"), + + 'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}), + ]; + } + + /** @inheritDoc */ + public function toString() { return "[Gitea|".$this->url."|".$this->filter."]"; } +} \ No newline at end of file diff --git a/www/extern/egg/RemoteSourceGithub.php b/www/extern/egg/RemoteSourceGithub.php new file mode 100644 index 0000000..459132c --- /dev/null +++ b/www/extern/egg/RemoteSourceGithub.php @@ -0,0 +1,171 @@ +url = $url; + $this->oauth_id = $oauth_id; + $this->oauth_secret = $oauth_secret; + $this->apitokenpath = $apitokenpath; + + if ($this->apitokenpath !== null && file_exists($this->apitokenpath)) + $this->apitoken = file_get_contents($this->apitokenpath); + else + $this->apitoken = null; + } + + /** + * @throws Exception + */ + public function queryAPIToken() { + $url = Utils::sharpFormat(self::URL_OAUTH_TOKEN, ['id'=>$this->oauth_id, 'secret'=>$this->oauth_secret, 'code'=>'egg']); + $fullresult = $result = file_get_contents($url); + + if (Utils::startsWith($fullresult, 'error=')) throw new EGGException('GitHub Auth failed: ' . $fullresult); + + $result = str_replace('access_token=', '', $result); + $result = str_replace('&scope=&token_type=bearer', '', $result); + + $this->logger->proclog("Updated Github API token"); + + if ($result!=='' && $result !== null && $this->apitokenpath !== null) + file_put_contents($this->apitokenpath, $result); + + $this->apitoken = $result; + } + + /** @inheritDoc */ + protected function preUpdate() + { + if ($this->apitoken === null) $this->queryAPIToken(); + } + + /** @inheritDoc */ + protected function postUpdate() + { + // + } + + /** @inheritDoc */ + protected function queryOrganizations($page) + { + throw new EGGException("Github does not support organization queries."); + } + + /** @inheritDoc */ + protected function queryRepositories($user, $page) + { + $url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page]); + return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); + } + + /** @inheritDoc */ + protected function queryBranches($reponame, $page) + { + $url = Utils::sharpFormat(self::API_BRANCHLIST, ['repo'=>$reponame, 'page'=>$page]); + return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); + } + + /** @inheritDoc */ + protected function queryCommits($reponame, $branchname, $startsha) + { + $url = Utils::sharpFormat(self::API_COMMITSLIST, [ 'repo'=>$reponame, 'sha'=>$startsha ]); + return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken); + } + + /** @inheritDoc */ + protected function readOrganization($data) + { + return + [ + 'id' => null, // no orgs in github + 'full_name' => null, // no orgs in github + 'key' => null, // no orgs in github + ]; + } + + /** @inheritDoc */ + protected function readRepository($data) + { + return + [ + 'full_name' => $data->{'full_name'}, + 'html_url' => $data->{'html_url'}, + ]; + } + + /** @inheritDoc */ + protected function readBranch($data) + { + if (isset($data->{'block'})) return null; + + return + [ + 'name' => $data->{'name'}, + 'sha' => $data->{'commit'}->{'sha'}, + ]; + } + + /** @inheritDoc */ + protected function readCommit($data) + { + return + [ + 'sha' => $data->{'sha'}, + + 'author_name' => $data->{'commit'}->{'author'}->{'name'}, + 'author_email' => $data->{'commit'}->{'author'}->{'email'}, + + 'committer_name' => $data->{'commit'}->{'committer'}->{'name'}, + 'committer_email' => $data->{'commit'}->{'committer'}->{'email'}, + + 'message' => $data->{'commit'}->{'message'}, + 'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"), + + 'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}), + ]; + } + + /** @inheritDoc */ + public function toString() { return "[Github|".$this->filter."]"; } +}