From a75053471d90c71c477134acca1a6c2383402ff3 Mon Sep 17 00:00:00 2001 From: Xavier Perret <xa.perret@outlook.com> Date: Wed, 19 Oct 2022 15:46:14 +0200 Subject: [PATCH] bim --- app/object-storage/object-storage.go | 75 +++++++++++++++---- readme-lab1.md | 105 +++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 14 deletions(-) create mode 100644 readme-lab1.md diff --git a/app/object-storage/object-storage.go b/app/object-storage/object-storage.go index 1f8b4c9..f4a4528 100644 --- a/app/object-storage/object-storage.go +++ b/app/object-storage/object-storage.go @@ -22,11 +22,6 @@ func addTransactionToBlobStorage(transaction Transaction, database Database, os return database } -func removeTransactionFromBlobStorage(transaction Transaction, database Database) Database { - - return database -} - func fakeTransaction(transactionToFake Transaction, fakeTransaction Transaction, database Database) Database { for i, trans := range database { if trans.Id == transactionToFake.Id { @@ -38,13 +33,6 @@ func fakeTransaction(transactionToFake Transaction, fakeTransaction Transaction, return database } -func writeDatabaseToFile(filepath string, db Database) { -} - -func readDatabaseFromFile(filepath string) Database { - return Database{} -} - func readDatabaseFromBlobStorage(blob Blob) Database { ctx := context.Background() @@ -75,6 +63,34 @@ func readDatabaseFromBlobStorage(blob Blob) Database { return database } +func readGivenBlobFromContainer(blob Blob, data any) any { + // Download the blob + ctx := context.Background() + get, err := blob.BlockBlobClient.Download(ctx, nil) + if err != nil { + log.Fatal(err) + } + + downloadedData := &bytes.Buffer{} + reader := get.Body(azblob.RetryReaderOptions{}) + _, err = downloadedData.ReadFrom(reader) + if err != nil { + log.Fatal(err) + } + err = reader.Close() + if err != nil { + log.Fatal(err) + } + + err = json.Unmarshal(downloadedData.Bytes(), &data) + if err != nil { + log.Fatal(err) + } + + fmt.Println(downloadedData.String()) + return data +} + func writeDatabaseToBlobStorage(database Database, blob Blob) { ctx := context.Background() data, err := json.Marshal(database) @@ -87,6 +103,18 @@ func writeDatabaseToBlobStorage(database Database, blob Blob) { } } +func writeGivenDataToBlob(blob Blob, data any) { + ctx := context.Background() + dataBytes, err := json.Marshal(data) + + // Upload to data to blob storage + _, err = blob.BlockBlobClient.UploadBufferToBlockBlob(ctx, dataBytes, azblob.HighLevelUploadToBlockBlobOption{}) + + if err != nil { + log.Fatalf("Failure to upload to blob: %+v", err) + } +} + func randomString() string { r := rand.New(rand.NewSource(time.Now().UnixNano())) return strconv.Itoa(r.Int()) @@ -118,11 +146,30 @@ func InitAzureCredentials(storageAccountName string) AzureCredentials { return newAzureCredentials } +func InitializeContainer(containerName string, azureCreds AzureCredentials) azblob.ContainerClient { + ctx := context.Background() + containerClient := azureCreds.ServiceClient.NewContainerClient(containerName) + _, err := containerClient.Create(ctx, nil) + if err != nil { + log.Fatal(err) + } + return containerClient +} + +func InitializeBlob(blobName string, azureCreds AzureCredentials, containerName string, containerClient azblob.ContainerClient) azblob.BlockBlobClient { + blobClient, err := azblob.NewBlockBlobClient(azureCreds.Url+containerName+"/"+blobName, azureCreds.Credential, nil) + if err != nil { + log.Fatal(err) + } + + return blobClient +} + func InitializeBlobFromObjectStorageCreds(blobName string, azureCreds AzureCredentials) Blob { ctx := context.Background() // Create the container - containerName := fmt.Sprintf("quickstart-%s", randomString()) + containerName := fmt.Sprintf("blockchain-storage-%s", randomString()) fmt.Printf("Creating a container named %s\n", containerName) containerClient := azureCreds.ServiceClient.NewContainerClient(containerName) _, err := containerClient.Create(ctx, nil) @@ -177,7 +224,7 @@ func TestObjectStorage() { ctx := context.Background() // url := "https://hepiadistributedsystems.blob.core.windows.net/" //replace <StorageAccountName> with your Azure storage account name azureCreds := InitAzureCredentials("hepiadistributedsystems") - blobName := "quickstartblob" + "-" + randomString() + blobName := "database" + "-" + randomString() objectStorage := InitializeBlobFromObjectStorageCreds(blobName, azureCreds) var db Database diff --git a/readme-lab1.md b/readme-lab1.md new file mode 100644 index 0000000..f5cae06 --- /dev/null +++ b/readme-lab1.md @@ -0,0 +1,105 @@ +# Distributed Systems + +## Lab 1 + +### Developing and deploying a broadcast algorithm for a blockchain management system on a Cloud infrastructure + +- Student : Xavier Perret +- Email : xavier.perret@etu.hesge.ch +- Group : 1 +- WARNING : D#1.2 is on branch "vite" and not on master +- <ssh://git@ssh.hesge.ch:10572/xavier.perret/perso-distributed-systems.git> +- <https://gitedu.hesge.ch/xavier.perret/perso-distributed-systems.git> +- Deliverable D#1.2 + +### Requirements + +- Go 1.14 +- Azure CLI +- Azure Storage Account configured (like <https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-go>) + - For testing you need to change the "hepiadistributedsystems" by the storage account name + ```go + azureCreds := InitAzureCredentials("hepiadistributedsystems") + ``` +- Should not be necessary _Azure Storage Blob Sdk (The right version is already in `go.mod` dependencies)_ + +### Methods Description + +```go +func addTransactionToBlobStorage(transaction Transaction, database Database, os Blob) Database +``` + +Add transaction to given database and upload it to the blob storage. The database is then returned. + +```go +func fakeTransaction(transactionToFake Transaction, fakeTransaction Transaction, database Database) Database +``` + +Find the given transaction to replace and replace it with the fake transaction. The database is then returned. + +```go +func readDatabaseFromBlobStorage(blob Blob) Database +``` + +Read (Download) the database from the blob storage and return it. + +```go +func readGivenBlobFromContainer(blob Blob, data any) any +``` + +Read given blob that is of type data from the container and return it. + +```go +func writeDatabaseToBlobStorage(database Database, blob Blob) +``` + +Write given database to given blob/file. + +```go +func writeGivenDataToBlob(blob Blob, data any) +``` + +Write given data to given blob/file. + +```go +func InitAzureCredentials(storageAccountName string) AzureCredentials +``` + +Initialize the Azure credentials with the given storage account name. + +```go +func InitializeBlobFromObjectStorageCreds(blobName string, azureCreds AzureCredentials) Blob +``` + +Initialize a container and then a blob/file (for upload/download) with the given blobName and returns it. + +```go +func InitializeContainer(containerName string, azureCreds AzureCredentials) azblob.ContainerClient +``` + +Initialize a container with the given containerName and returns it (to create/delete/manage blob in it). + +```go +func InitializeBlob(blobName string, azureCreds AzureCredentials, containerName string, containerClient azblob.ContainerClient) azblob.BlockBlobClient +``` + +Initialize a blob/file (for upload/download) with the given blobName and returns an object to interact with it (write/read data). + +```go +func ListBlobs(blob Blob) +``` + +List blobs in container + +```go +func PrintingDatabaseToConsole(database Database) +``` + +Print the database to the console. + +```go +func TestObjectStorage() +``` + +Provide tests for the object storage. + -- GitLab