From d3aaa7e6a60f7797ed5534925ce84a9c7bfede77 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 10 Aug 2023 10:47:41 +0100 Subject: [PATCH 1/4] Add isset checks for 0 data received on certain endpoints --- src/Migration/Sources/Firebase.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 63884fa..5883bc8 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -181,6 +181,10 @@ private function exportUsers(int $batchSize) 'Content-Type' => 'application/json', ], $request); + if (! isset($response['users'])) { + break; + } + $result = $response['users']; $nextPageToken = $response['nextPageToken'] ?? null; @@ -261,6 +265,10 @@ private function exportDB(int $batchSize, bool $pushDocuments, Database $databas 'pageToken' => $nextPageToken, ]); + if (! isset($result['collectionIds'])) { + break; + } + // Transfer Collections foreach ($result['collectionIds'] as $collection) { $collections[] = new Collection($database, $collection, $collection); @@ -464,6 +472,10 @@ private function exportBuckets(int $batchsize) break; } + if (! isset($result['items'])) { + break; + } + foreach ($result['items'] as $bucket) { $this->callback([new Bucket($bucket['id'], $bucket['name'], [], false)]); } From 0b39e25683d7368a96bdd99437860012c15f382f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 11 Aug 2023 13:29:13 +0100 Subject: [PATCH 2/4] Update Firebase.php --- src/Migration/Sources/Firebase.php | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 5883bc8..f61e5fa 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -147,6 +147,8 @@ public function report(array $resources = []): array throw new \Exception('Datastore Scope Missing'); } + $this->exportGroupDatabases(1000, [Resource::TYPE_DATABASE, Resource::TYPE_COLLECTION, Resource::TYPE_ATTRIBUTE, Resource::TYPE_DOCUMENT]); + return []; } @@ -258,14 +260,27 @@ private function exportDB(int $batchSize, bool $pushDocuments, Database $databas while (true) { $collections = []; - $result = $this->call('POST', $baseURL.':listCollectionIds', [ - 'Content-Type' => 'application/json', - ], [ - 'pageSize' => $batchSize, - 'pageToken' => $nextPageToken, - ]); + try { + $result = $this->call('POST', $baseURL.':listCollectionIds', [ + 'Content-Type' => 'application/json', + ], [ + 'pageSize' => $batchSize, + 'pageToken' => $nextPageToken, + ]); + + if (! isset($result['collectionIds'])) { + break; + } + } catch (\Exception $e) { + if ($e->getCode() == 403) { + $errorMessage = new Collection($database, 'firestore', 'firestore'); + + $errorMessage->setStatus(Resource::STATUS_ERROR); + $errorMessage->setMessage($e->getMessage()); + + $this->cache->add($errorMessage); + } - if (! isset($result['collectionIds'])) { break; } From 957b7cb2a787cf07000321413e125c821d1d17e4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 11 Aug 2023 14:58:22 +0100 Subject: [PATCH 3/4] Update Firebase.php --- src/Migration/Sources/Firebase.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index f61e5fa..713f08f 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -147,8 +147,6 @@ public function report(array $resources = []): array throw new \Exception('Datastore Scope Missing'); } - $this->exportGroupDatabases(1000, [Resource::TYPE_DATABASE, Resource::TYPE_COLLECTION, Resource::TYPE_ATTRIBUTE, Resource::TYPE_DOCUMENT]); - return []; } From 82660c469cca13a020a7254470295dadf1fdb56b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 15 Aug 2023 17:08:50 +0100 Subject: [PATCH 4/4] Add checks for if services are disabled --- playground.php | 16 +++++----- src/Migration/Sources/Firebase.php | 48 +++++++++++++++++++++++++++++- src/Migration/Transfer.php | 17 +++++------ 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/playground.php b/playground.php index 5200415..9112993 100644 --- a/playground.php +++ b/playground.php @@ -69,25 +69,23 @@ * Initialise Transfer Class */ $transfer = new Transfer( - $sourceSupabase, + $sourceFirebase, $destinationLocal ); $sourceFirebase->report(); -// $sourceSupabase->report(); - // /** // * Run Transfer // */ -// $transfer->run($sourceAppwrite->getSupportedResources(), -// function (array $resources) { -// } -// ); +$transfer->run($sourceFirebase->getSupportedResources(), + function (array $resources) { + } +); -// $report = []; +$report = []; -// $cache = $transfer->getCache()->getAll(); +$cache = $transfer->getCache()->getAll(); // foreach ($cache as $type => $resources) { // foreach ($resources as $resource) { diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 713f08f..8b7c51f 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -152,6 +152,20 @@ public function report(array $resources = []): array protected function exportGroupAuth(int $batchSize, array $resources) { + // Check if Auth is enabled + try { + $this->call('GET', 'https://identitytoolkit.googleapis.com/v1/projects'); + } catch (\Exception $e) { + $message = json_decode($e->getMessage(), true); + + if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') { + // IdentityKit is disabled + return; + } + + throw $e; + } + if (in_array(Resource::TYPE_USER, $resources)) { $this->exportUsers($batchSize); } @@ -238,6 +252,20 @@ private function calculateUserType(array $providerData): array protected function exportGroupDatabases(int $batchSize, array $resources) { + // Check if Firestore is enabled + try { + $this->call('GET', 'https://firestore.googleapis.com/v1/projects/'.$this->projectID.'/databases'); + } catch (\Exception $e) { + $message = json_decode($e->getMessage(), true); + + if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') { + // Firestore is disabled + return; + } + + throw $e; + } + if (in_array(Resource::TYPE_DATABASE, $resources)) { $database = new Database('default', 'default'); $database->setOriginalId('(default)'); @@ -265,7 +293,7 @@ private function exportDB(int $batchSize, bool $pushDocuments, Database $databas 'pageSize' => $batchSize, 'pageToken' => $nextPageToken, ]); - + if (! isset($result['collectionIds'])) { break; } @@ -458,6 +486,24 @@ private function convertDocument(Collection $collection, array $document): Docum protected function exportGroupStorage(int $batchSize, array $resources) { + // Check if storage is enabled + try { + $this->call('GET', 'https://storage.googleapis.com/storage/v1/b', [], [ + 'project' => $this->projectID, + 'maxResults' => 1, + 'alt' => 'json', + ]); + } catch (\Exception $e) { + $message = json_decode($e->getMessage(), true); + + if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') { + // Storage is disabled + return; + } + + throw $e; + } + if (in_array(Resource::TYPE_BUCKET, $resources)) { $this->exportBuckets($batchSize); } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 6c19a34..ce59c64 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -88,19 +88,18 @@ public function getStatusCounters() ]; } - if ($this->source->previousReport) { - foreach ($this->source->previousReport as $resource => $data) { - if ($resource != 'size' && $resource != 'version') { - $status[$resource]['pending'] = $data; - } - } - } - foreach ($this->cache->getAll() as $resources) { foreach ($resources as $resource) { /** @var resource $resource */ $status[$resource->getName()][$resource->getStatus()]++; - $status[$resource->getName()]['pending']--; + } + } + + if ($this->source->previousReport) { + foreach ($this->source->previousReport as $resource => $data) { + if ($resource != 'size' && $resource != 'version' && isset($status[$resource])) { + $status[$resource]['pending'] = $data; + } } }