1
0

support for '*/*' filter in egg (gitea only)
All checks were successful
Build Docker and Deploy / Build Docker (push) Successful in 1m36s
Build Docker and Deploy / Deploy to Server (push) Successful in 18s

This commit is contained in:
Mike Schwörer 2025-07-12 15:42:22 +02:00
parent ba579dab23
commit ee7debde55
Signed by: Mikescher
GPG Key ID: D3C7172E0A70F8CF
4 changed files with 384 additions and 287 deletions

View File

@ -3,6 +3,8 @@
require_once 'Logger.php';
require_once 'EGGException.php';
require_once 'RemoteSource.php';
require_once 'RemoteSourceGithub.php';
require_once 'RemoteSourceGitea.php';
require_once 'OutputGenerator.php';
require_once 'EGGDatabase.php';
require_once 'Utils.php';

View File

@ -115,6 +115,12 @@ abstract class StandardGitConnection implements IRemoteSource
*/
protected abstract function postUpdate();
/**
* @param int $page
* @return array
*/
protected abstract function queryOrganizations($page);
/**
* @param string $user
* @param int $page
@ -137,6 +143,13 @@ abstract class StandardGitConnection implements IRemoteSource
*/
protected abstract function queryCommits($reponame, $branchname, $startsha);
/**
* @param mixed $data
* @return array
* @throws Exception
*/
protected abstract function readOrganization($data);
/**
* @param mixed $data
* @return array
@ -168,8 +181,46 @@ abstract class StandardGitConnection implements IRemoteSource
$result = [];
if ($f[0] === '*') {
$page = 1;
$json = $this->queryRepositories($f[0], $page);
$json = $this->queryOrganizations($page);
while (! empty($json)) {
foreach ($json as $result_org) {
$jdata = $this->readOrganization($result_org);
$this->logger->proclog("Found Org in Remote: " . $jdata['full_name'] . " (" . $jdata['key'] . ")");
$r0 = $this->listAndUpdateRepositoriesOfOrganization($db, $jdata['key']);
$result = array_merge($result, $r0);
}
$page++;
$json = $this->queryOrganizations($page);
}
} else {
$r0 = $this->listAndUpdateRepositoriesOfOrganization($db, $f[0]);
$result = array_merge($result, $r0);
}
$db->deleteOtherRepositories($this->name, $result);
return $result;
}
private function listAndUpdateRepositoriesOfOrganization(EGGDatabase $db, string $org){
$result = [];
$page = 1;
$json = $this->queryRepositories($org, $page);
$found = [];
while (! empty($json))
@ -197,11 +248,9 @@ abstract class StandardGitConnection implements IRemoteSource
if ($count === 0) break;
$page++;
$json = $this->queryRepositories($f[0], $page);
$json = $this->queryRepositories($org, $page);
}
$db->deleteOtherRepositories($this->name, $result);
return $result;
}
@ -412,264 +461,3 @@ abstract class StandardGitConnection implements IRemoteSource
/** @inheritDoc */
public abstract function toString();
}
class GithubConnection extends StandardGitConnection
{
const API_OAUTH_AUTH = 'https://github.com/login/oauth/authorize?client_id=%s';
const URL_OAUTH_TOKEN = 'https://github.com/login/oauth/access_token?client_id={id}&client_secret={secret}&code={code}';
const API_RATELIMIT = 'https://api.github.com/rate_limit';
const API_REPOSITORIESLIST = 'https://api.github.com/users/{user}/repos?page={page}&per_page=100';
const API_COMMITSLIST = 'https://api.github.com/repos/{repo}/commits?per_page=100&sha={sha}';
const API_BRANCHLIST = 'https://api.github.com/repos/{repo}/branches?page={page}';
/** @var string $url */
private $url;
/** @var string $oauth_id */
private $oauth_id;
/** @var string $oauth_secret */
private $oauth_secret;
/** @var string $apitokenpath */
private $apitokenpath;
/** @var string $apitoken */
private $apitoken;
/**
* @param ILogger $logger
* @param string $name
* @param string $url
* @param string $filter
* @param string[] exclusions
* @param string $oauth_id
* @param string $oauth_secret
* @param string $apitokenpath
*/
public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $oauth_id, string $oauth_secret, string $apitokenpath)
{
parent::__construct($logger, $name, $filter, $exclusions);
$this->url = $url;
$this->oauth_id = $oauth_id;
$this->oauth_secret = $oauth_secret;
$this->apitokenpath = $apitokenpath;
if ($this->apitokenpath !== null && file_exists($this->apitokenpath))
$this->apitoken = file_get_contents($this->apitokenpath);
else
$this->apitoken = null;
}
/**
* @throws Exception
*/
public function queryAPIToken() {
$url = Utils::sharpFormat(self::URL_OAUTH_TOKEN, ['id'=>$this->oauth_id, 'secret'=>$this->oauth_secret, 'code'=>'egg']);
$fullresult = $result = file_get_contents($url);
if (Utils::startsWith($fullresult, 'error=')) throw new EGGException('GitHub Auth failed: ' . $fullresult);
$result = str_replace('access_token=', '', $result);
$result = str_replace('&scope=&token_type=bearer', '', $result);
$this->logger->proclog("Updated Github API token");
if ($result!=='' && $result !== null && $this->apitokenpath !== null)
file_put_contents($this->apitokenpath, $result);
$this->apitoken = $result;
}
/** @inheritDoc */
protected function preUpdate()
{
if ($this->apitoken === null) $this->queryAPIToken();
}
/** @inheritDoc */
protected function postUpdate()
{
//
}
/** @inheritDoc */
protected function queryRepositories($user, $page)
{
$url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page]);
return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken);
}
/** @inheritDoc */
protected function queryBranches($reponame, $page)
{
$url = Utils::sharpFormat(self::API_BRANCHLIST, ['repo'=>$reponame, 'page'=>$page]);
return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken);
}
/** @inheritDoc */
protected function queryCommits($reponame, $branchname, $startsha)
{
$url = Utils::sharpFormat(self::API_COMMITSLIST, [ 'repo'=>$reponame, 'sha'=>$startsha ]);
return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken);
}
/** @inheritDoc */
protected function readRepository($data)
{
return
[
'full_name' => $data->{'full_name'},
'html_url' => $data->{'html_url'},
];
}
/** @inheritDoc */
protected function readBranch($data)
{
if (isset($data->{'block'})) return null;
return
[
'name' => $data->{'name'},
'sha' => $data->{'commit'}->{'sha'},
];
}
/** @inheritDoc */
protected function readCommit($data)
{
return
[
'sha' => $data->{'sha'},
'author_name' => $data->{'commit'}->{'author'}->{'name'},
'author_email' => $data->{'commit'}->{'author'}->{'email'},
'committer_name' => $data->{'commit'}->{'committer'}->{'name'},
'committer_email' => $data->{'commit'}->{'committer'}->{'email'},
'message' => $data->{'commit'}->{'message'},
'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"),
'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}),
];
}
/** @inheritDoc */
public function toString() { return "[Github|".$this->filter."]"; }
}
class GiteaConnection extends StandardGitConnection
{
const API_BASE_URL = '/api/v1';
const API_USER_REPO_LIST = '/users/{user}/repos?page={page}&limit={limit}';
const API_BRANCH_LIST = '/repos/{repo}/branches?page={page}&limit={limit}';
const API_COMMIT_LIST = '/repos/{repo}/commits?sha={sha}&limit={limit}';
/** @var string $url */
private $url;
/** @var string $username */
private $username;
/** @var string $password */
private $password;
/**
* @param ILogger $logger
* @param string $name
* @param string $url
* @param string $filter
* @param string[] $exclusions
* @param string $username
* @param string $password
*/
public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $username, string $password)
{
parent::__construct($logger, $name, $filter, $exclusions);
$this->url = $url;
$this->username = $username;
$this->password = $password;
}
/** @inheritDoc */
protected function preUpdate()
{
//
}
/** @inheritDoc */
protected function postUpdate()
{
//
}
/** @inheritDoc */
protected function queryRepositories($user, $page)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_REPO_LIST), ['user'=>$user, 'page'=>$page, 'limit'=>64 ]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function queryBranches($reponame, $page)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_BRANCH_LIST), ['repo'=>$reponame, 'page'=>$page, 'limit'=>64]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function queryCommits($reponame, $branchname, $startsha)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_COMMIT_LIST), [ 'repo'=>$reponame, 'sha'=>$startsha, 'limit'=>1024 ]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function readRepository($data)
{
return
[
'full_name' => $data->{'full_name'},
'html_url' => $data->{'html_url'},
];
}
/** @inheritDoc */
protected function readBranch($data)
{
return
[
'name' => $data->{'name'},
'sha' => $data->{'commit'}->{'id'},
];
}
/** @inheritDoc */
protected function readCommit($data)
{
return
[
'sha' => $data->{'commit'}->{'tree'}->{'sha'},
'author_name' => $data->{'commit'}->{'author'}->{'name'},
'author_email' => $data->{'commit'}->{'author'}->{'email'},
'committer_name' => $data->{'commit'}->{'committer'}->{'name'},
'committer_email' => $data->{'commit'}->{'committer'}->{'email'},
'message' => $data->{'commit'}->{'message'},
'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"),
'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}),
];
}
/** @inheritDoc */
public function toString() { return "[Gitea|".$this->url."|".$this->filter."]"; }
}

136
www/extern/egg/RemoteSourceGitea.php vendored Normal file
View File

@ -0,0 +1,136 @@
<?php
require_once 'Utils.php';
require_once 'EGGDatabase.php';
require_once 'RemoteSource.php';
class GiteaConnection extends StandardGitConnection
{
const API_BASE_URL = '/api/v1';
const API_USER_REPO_LIST = '/users/{user}/repos?page={page}&limit={limit}';
const API_USER_ORG_LIST = '/orgs?page={page}&limit={limit}';
const API_BRANCH_LIST = '/repos/{repo}/branches?page={page}&limit={limit}';
const API_COMMIT_LIST = '/repos/{repo}/commits?sha={sha}&limit={limit}';
/** @var string $url */
private $url;
/** @var string $username */
private $username;
/** @var string $password */
private $password;
/**
* @param ILogger $logger
* @param string $name
* @param string $url
* @param string $filter
* @param string[] $exclusions
* @param string $username
* @param string $password
*/
public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $username, string $password)
{
parent::__construct($logger, $name, $filter, $exclusions);
$this->url = $url;
$this->username = $username;
$this->password = $password;
}
/** @inheritDoc */
protected function preUpdate()
{
//
}
/** @inheritDoc */
protected function postUpdate()
{
//
}
/** @inheritDoc */
protected function queryOrganizations($page)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_ORG_LIST), ['page'=>$page, 'limit'=>64 ]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function queryRepositories($user, $page)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_USER_REPO_LIST), ['user'=>$user, 'page'=>$page, 'limit'=>64 ]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function queryBranches($reponame, $page)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_BRANCH_LIST), ['repo'=>$reponame, 'page'=>$page, 'limit'=>64]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function queryCommits($reponame, $branchname, $startsha)
{
$url = Utils::sharpFormat(Utils::urlCombine($this->url, self::API_BASE_URL, self::API_COMMIT_LIST), [ 'repo'=>$reponame, 'sha'=>$startsha, 'limit'=>1024 ]);
return Utils::getJSONWithTokenBasicAuth($this->logger, $url, $this->username, $this->password);
}
/** @inheritDoc */
protected function readRepository($data)
{
return
[
'full_name' => $data->{'full_name'},
'html_url' => $data->{'html_url'},
];
}
/** @inheritDoc */
protected function readOrganization($data)
{
return
[
'id' => $data->{'id'},
'full_name' => $data->{'full_name'},
'key' => $data->{'username'},
];
}
/** @inheritDoc */
protected function readBranch($data)
{
return
[
'name' => $data->{'name'},
'sha' => $data->{'commit'}->{'id'},
];
}
/** @inheritDoc */
protected function readCommit($data)
{
return
[
'sha' => $data->{'commit'}->{'tree'}->{'sha'},
'author_name' => $data->{'commit'}->{'author'}->{'name'},
'author_email' => $data->{'commit'}->{'author'}->{'email'},
'committer_name' => $data->{'commit'}->{'committer'}->{'name'},
'committer_email' => $data->{'commit'}->{'committer'}->{'email'},
'message' => $data->{'commit'}->{'message'},
'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"),
'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}),
];
}
/** @inheritDoc */
public function toString() { return "[Gitea|".$this->url."|".$this->filter."]"; }
}

171
www/extern/egg/RemoteSourceGithub.php vendored Normal file
View File

@ -0,0 +1,171 @@
<?php
require_once 'Utils.php';
require_once 'EGGDatabase.php';
require_once 'RemoteSource.php';
class GithubConnection extends StandardGitConnection
{
const API_OAUTH_AUTH = 'https://github.com/login/oauth/authorize?client_id=%s';
const URL_OAUTH_TOKEN = 'https://github.com/login/oauth/access_token?client_id={id}&client_secret={secret}&code={code}';
const API_RATELIMIT = 'https://api.github.com/rate_limit';
const API_REPOSITORIESLIST = 'https://api.github.com/users/{user}/repos?page={page}&per_page=100';
const API_COMMITSLIST = 'https://api.github.com/repos/{repo}/commits?per_page=100&sha={sha}';
const API_BRANCHLIST = 'https://api.github.com/repos/{repo}/branches?page={page}';
/** @var string $url */
private $url;
/** @var string $oauth_id */
private $oauth_id;
/** @var string $oauth_secret */
private $oauth_secret;
/** @var string $apitokenpath */
private $apitokenpath;
/** @var string $apitoken */
private $apitoken;
/**
* @param ILogger $logger
* @param string $name
* @param string $url
* @param string $filter
* @param string[] exclusions
* @param string $oauth_id
* @param string $oauth_secret
* @param string $apitokenpath
*/
public function __construct(ILogger $logger, string $name, string $url, string $filter, array $exclusions, string $oauth_id, string $oauth_secret, string $apitokenpath)
{
parent::__construct($logger, $name, $filter, $exclusions);
$this->url = $url;
$this->oauth_id = $oauth_id;
$this->oauth_secret = $oauth_secret;
$this->apitokenpath = $apitokenpath;
if ($this->apitokenpath !== null && file_exists($this->apitokenpath))
$this->apitoken = file_get_contents($this->apitokenpath);
else
$this->apitoken = null;
}
/**
* @throws Exception
*/
public function queryAPIToken() {
$url = Utils::sharpFormat(self::URL_OAUTH_TOKEN, ['id'=>$this->oauth_id, 'secret'=>$this->oauth_secret, 'code'=>'egg']);
$fullresult = $result = file_get_contents($url);
if (Utils::startsWith($fullresult, 'error=')) throw new EGGException('GitHub Auth failed: ' . $fullresult);
$result = str_replace('access_token=', '', $result);
$result = str_replace('&scope=&token_type=bearer', '', $result);
$this->logger->proclog("Updated Github API token");
if ($result!=='' && $result !== null && $this->apitokenpath !== null)
file_put_contents($this->apitokenpath, $result);
$this->apitoken = $result;
}
/** @inheritDoc */
protected function preUpdate()
{
if ($this->apitoken === null) $this->queryAPIToken();
}
/** @inheritDoc */
protected function postUpdate()
{
//
}
/** @inheritDoc */
protected function queryOrganizations($page)
{
throw new EGGException("Github does not support organization queries.");
}
/** @inheritDoc */
protected function queryRepositories($user, $page)
{
$url = Utils::sharpFormat(self::API_REPOSITORIESLIST, ['user'=>$user, 'page'=>$page]);
return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken);
}
/** @inheritDoc */
protected function queryBranches($reponame, $page)
{
$url = Utils::sharpFormat(self::API_BRANCHLIST, ['repo'=>$reponame, 'page'=>$page]);
return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken);
}
/** @inheritDoc */
protected function queryCommits($reponame, $branchname, $startsha)
{
$url = Utils::sharpFormat(self::API_COMMITSLIST, [ 'repo'=>$reponame, 'sha'=>$startsha ]);
return Utils::getJSONWithTokenAuth($this->logger, $url, $this->apitoken);
}
/** @inheritDoc */
protected function readOrganization($data)
{
return
[
'id' => null, // no orgs in github
'full_name' => null, // no orgs in github
'key' => null, // no orgs in github
];
}
/** @inheritDoc */
protected function readRepository($data)
{
return
[
'full_name' => $data->{'full_name'},
'html_url' => $data->{'html_url'},
];
}
/** @inheritDoc */
protected function readBranch($data)
{
if (isset($data->{'block'})) return null;
return
[
'name' => $data->{'name'},
'sha' => $data->{'commit'}->{'sha'},
];
}
/** @inheritDoc */
protected function readCommit($data)
{
return
[
'sha' => $data->{'sha'},
'author_name' => $data->{'commit'}->{'author'}->{'name'},
'author_email' => $data->{'commit'}->{'author'}->{'email'},
'committer_name' => $data->{'commit'}->{'committer'}->{'name'},
'committer_email' => $data->{'commit'}->{'committer'}->{'email'},
'message' => $data->{'commit'}->{'message'},
'date' => (new DateTime($data->{'commit'}->{'author'}->{'date'}))->format("Y-m-d H:i:s"),
'parents' => array_map(function ($v){ return $v->{'sha'}; }, $data->{'parents'}),
];
}
/** @inheritDoc */
public function toString() { return "[Github|".$this->filter."]"; }
}