Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat-eldad2-coroutines #7901

Open
wants to merge 39 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
d8996b4
Implement DB proxy
Meldiron Mar 5, 2024
e807a5d
Upgrade DB proxy
Meldiron Mar 5, 2024
a9731cd
Fix account tests
Meldiron Mar 6, 2024
882d8e5
Fix database tests
Meldiron Mar 6, 2024
f83d6e6
PR review changes
Meldiron Mar 6, 2024
4fa9ccd
upgrade libs
Meldiron Mar 6, 2024
1b65c5f
linter fix
Meldiron Mar 6, 2024
27e995e
Reomve leftpover
Meldiron Mar 6, 2024
ee065bc
Merge branch '1.5.x' into feat-database-proxy
Meldiron Mar 7, 2024
8a8638a
Formatting fix
Meldiron Mar 7, 2024
5a7c43a
Introduce code analysis
Meldiron Mar 7, 2024
efeb898
Import fixes
Meldiron Mar 7, 2024
005a239
Auth fixes
Meldiron Mar 7, 2024
db16748
Finish fixing code QL warnings
Meldiron Mar 8, 2024
d5f180d
Add debugging
Meldiron Mar 8, 2024
513bf7e
Update .env
Meldiron Mar 8, 2024
4de9591
Disable debug
Meldiron Mar 8, 2024
2a0a69f
Fix connections pool
Meldiron Mar 9, 2024
713928c
Couroutines test
eldadfux Apr 1, 2024
6ce8781
Fixed email escaping
eldadfux Apr 1, 2024
c73ef2c
Fixed 451 status code
eldadfux Apr 1, 2024
b570368
Fixed auth error in users usage
eldadfux Apr 1, 2024
5ce3de4
Merge remote-tracking branch 'origin/main' into feat-eldad2-coroutines
eldadfux Apr 2, 2024
be7a6e5
Fixed whitespace
eldadfux Apr 2, 2024
378bb5f
Fixed server
eldadfux Apr 2, 2024
a1db69b
Fixed missing System namespace
eldadfux Apr 2, 2024
bd57955
Enabled coroutines
eldadfux Apr 3, 2024
211bcf8
Removed proxy db
eldadfux Apr 9, 2024
08b3182
WIP
eldadfux Apr 14, 2024
9e234b7
updated server
eldadfux Apr 14, 2024
6cade89
New init structure
eldadfux Apr 14, 2024
766b2ba
Avatars tests are green!
eldadfux Apr 14, 2024
fd2410d
Fixed bootstrap
eldadfux Apr 14, 2024
a5ba262
Fixed bootstrap file
eldadfux Apr 15, 2024
49a50b3
Test DB failure
eldadfux Apr 15, 2024
87ca2df
Applied fixes for tests
eldadfux Apr 15, 2024
13eb3bc
Fixed users tests
eldadfux Apr 15, 2024
07a830e
Work on the main worker
eldadfux Apr 15, 2024
2e4c6ea
Merge branch 'main' into feat-eldad2-coroutines
eldadfux Apr 15, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/workflows/codeql-phpstan.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
name: "CodeQL"

on: [pull_request]
jobs:
lint:
name: CodeQL
runs-on: ubuntu-latest

steps:
- name: Check out the repo
uses: actions/checkout@v2

- name: Run CodeQL
run: |
docker run --rm -v $PWD:/app composer sh -c \
"composer install --profile --ignore-platform-reqs && composer check"
2 changes: 1 addition & 1 deletion CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@
* Bump console to version 3.2.7 [#7148](https://github.com/appwrite/appwrite/pull/7148)
* Chore update database to 0.45.2 [#7138](https://github.com/appwrite/appwrite/pull/7138)
* Implement queue thresholds for the health API [#7123](https://github.com/appwrite/appwrite/pull/7123)
* Add Authorization::skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124)
* Add $auth->skip to the usage worker [#7124](https://github.com/appwrite/appwrite/pull/7124)

## Bug fixes
* fix: use queueForDeletes in git installation delete endpoint [#7140](https://github.com/appwrite/appwrite/pull/7140)
Expand Down
75 changes: 50 additions & 25 deletions app/cli.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
use Appwrite\Event\Func;
use Appwrite\Event\Hamster;
use Appwrite\Platform\Appwrite;
use Appwrite\Utopia\Queue\Connections;
use Utopia\Cache\Adapter\Sharding;
use Utopia\Cache\Cache;
use Utopia\CLI\CLI;
Expand All @@ -16,52 +17,56 @@
use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Validator\Authorization;
use Utopia\Http\Http;
use Utopia\Logger\Log;
use Utopia\Platform\Service;
use Utopia\Pools\Group;
use Utopia\Queue\Connection;
use Utopia\Registry\Registry;
use Utopia\System\System;

Authorization::disable();
global $register;

CLI::setResource('register', fn () => $register);

CLI::setResource('cache', function ($pools) {
CLI::setResource('connections', function () {
return new Connections();
});

CLI::setResource('cache', function ($pools, Connections $connections) {
$list = Config::getParam('pools-cache', []);
$adapters = [];

foreach ($list as $value) {
$adapters[] = $pools
->get($value)
->pop()
->getResource()
;
$connection = $pools->get($value)->pop();
$connections->add($connection);
$adapters[] = $connection->getResource();
}

return new Cache(new Sharding($adapters));
}, ['pools']);
}, ['pools', 'connections']);

CLI::setResource('pools', function (Registry $register) {
return $register->get('pools');
}, ['register']);

CLI::setResource('dbForConsole', function ($pools, $cache) {
CLI::setResource('dbForConsole', function ($pools, $cache, $auth, Connections $connections) {
$sleep = 3;
$maxAttempts = 5;
$attempts = 0;
$ready = false;

$connection = null;

do {
$attempts++;
try {
// Prepare database connection
$dbAdapter = $pools
->get('console')
->pop()
->getResource();
$connection = $pools->get('console')->pop();
$dbAdapter = $connection->getResource();

$dbForConsole = new Database($dbAdapter, $cache);
$dbForConsole->setAuthorization($auth);

$dbForConsole
->setNamespace('_console')
Expand All @@ -78,23 +83,31 @@

$ready = true;
} catch (\Throwable $err) {
if($connection !== null) {
$connection->reclaim();
$connection = null;
}

Console::warning($err->getMessage());
$pools->get('console')->reclaim();
sleep($sleep);
}
} while ($attempts < $maxAttempts && !$ready);

if($connection !== null) {
$connections->add($connection);
}

if (!$ready) {
throw new Exception("Console is not ready yet. Please try again later.");
}

return $dbForConsole;
}, ['pools', 'cache']);
}, ['pools', 'cache', 'auth', 'connections']);

CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache) {
CLI::setResource('getProjectDB', function (Group $pools, Database $dbForConsole, $cache, $auth, Connections $connections) {
$databases = []; // TODO: @Meldiron This should probably be responsibility of utopia-php/pools

return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases) {
return function (Document $project) use ($pools, $dbForConsole, $cache, &$databases, $auth, $connections) {
if ($project->isEmpty() || $project->getId() === 'console') {
return $dbForConsole;
}
Expand All @@ -107,12 +120,12 @@
return $database;
}

$dbAdapter = $pools
->get($databaseName)
->pop()
->getResource();
$connection = $pools->get($databaseName)->pop();
$connections->add($connection);
$dbAdapter = $connection->getResource();

$database = new Database($dbAdapter, $cache);
$database->setAuthorization($auth);

$databases[$databaseName] = $database;

Expand All @@ -123,11 +136,13 @@

return $database;
};
}, ['pools', 'dbForConsole', 'cache']);
}, ['pools', 'dbForConsole', 'cache', 'auth', 'connections']);

CLI::setResource('queue', function (Group $pools) {
return $pools->get('queue')->pop()->getResource();
}, ['pools']);
CLI::setResource('queue', function (Group $pools, Connections $connections) {
$connection = $pools->get('queue')->pop();
$connections->add($connection);
return $connection->getResource();
}, ['pools', 'connections']);
CLI::setResource('queueForFunctions', function (Connection $queue) {
return new Func($queue);
}, ['queue']);
Expand Down Expand Up @@ -165,6 +180,7 @@
$log->setAction($action);

$isProduction = System::getEnv('_APP_ENV', 'development') === 'production';

$log->setEnvironment($isProduction ? Log::ENVIRONMENT_PRODUCTION : Log::ENVIRONMENT_STAGING);

$responseCode = $logger->addLog($log);
Expand All @@ -176,11 +192,20 @@
};
}, ['register']);

CLI::setResource('auth', fn () => new Authorization());

$platform = new Appwrite();
$platform->init(Service::TYPE_CLI);

$cli = $platform->getCli();

$cli
->init()
->inject('auth')
->action(function (Authorization $auth) {
$auth->disable();
});

$cli
->error()
->inject('error')
Expand Down