From d3977be9485701706d6bae6fc3c98353de2a3a7e Mon Sep 17 00:00:00 2001 From: scriptcamp Date: Tue, 20 Dec 2022 18:18:37 +0530 Subject: [PATCH 01/10] DynamoDB Update --- boto3/dynamodb/README.json | 5 + boto3/dynamodb/createTable.py | 33 ++ boto3/dynamodb/data.json | 492 +++++++++++++++++++++++ boto3/dynamodb/loadData.py | 26 ++ boto3/dynamodb/query.py | 26 ++ boto3/{ => ec2}/create-instance.py | 0 generic/exception-handling.py | 19 - operating-system/README.md | 7 + operating-system/directory-operations.py | 7 - operating-system/file-operations.py | 0 operating-system/shell-execution.py | 2 - 11 files changed, 589 insertions(+), 28 deletions(-) create mode 100644 boto3/dynamodb/README.json create mode 100644 boto3/dynamodb/createTable.py create mode 100644 boto3/dynamodb/data.json create mode 100644 boto3/dynamodb/loadData.py create mode 100644 boto3/dynamodb/query.py rename boto3/{ => ec2}/create-instance.py (100%) create mode 100644 operating-system/README.md delete mode 100644 operating-system/directory-operations.py delete mode 100644 operating-system/file-operations.py diff --git a/boto3/dynamodb/README.json b/boto3/dynamodb/README.json new file mode 100644 index 0000000..494ec41 --- /dev/null +++ b/boto3/dynamodb/README.json @@ -0,0 +1,5 @@ +## Working With DynamoDB Using Boto3 + +This example shows loading sample data to dynamoDB and using boto3 to acess the data from the table. + +Code Credits: [fernandomc.com Guide](https://www.fernandomc.com/posts/ten-examples-of-getting-data-from-dynamodb-with-python-and-boto3/) \ No newline at end of file diff --git a/boto3/dynamodb/createTable.py b/boto3/dynamodb/createTable.py new file mode 100644 index 0000000..df7c604 --- /dev/null +++ b/boto3/dynamodb/createTable.py @@ -0,0 +1,33 @@ +import boto3 + +dynamodb = boto3.client("dynamodb") + +response = dynamodb.create_table( + TableName="basicSongsTable", + AttributeDefinitions=[ + { + "AttributeName": "artist", + "AttributeType": "S" + }, + { + "AttributeName": "song", + "AttributeType": "S" + } + ], + KeySchema=[ + { + "AttributeName": "artist", + "KeyType": "HASH" + }, + { + "AttributeName": "song", + "KeyType": "RANGE" + } + ], + ProvisionedThroughput={ + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1 + } +) + +print(response) \ No newline at end of file diff --git a/boto3/dynamodb/data.json b/boto3/dynamodb/data.json new file mode 100644 index 0000000..aecf5a3 --- /dev/null +++ b/boto3/dynamodb/data.json @@ -0,0 +1,492 @@ +[ + { + "artist": "Romero Allen", + "song": "Atomic Dim", + "id": "b4b0da3f-36e3-4569-b196-3ad982f72bbd", + "priceUsdCents": 392, + "publisher": "QUAREX" + }, + { + "artist": "Hilda Barnes", + "song": "Almond Dutch", + "id": "eeb58c73-603f-4d6b-9e3b-cf587488f488", + "priceUsdCents": 161, + "publisher": "LETPRO" + }, + { + "artist": "Hall Ramos", + "song": "Carmine Drab", + "id": "951637bd-59ef-4448-93f5-e179bfbd20a0", + "priceUsdCents": 370, + "publisher": "BRAINQUIL" + }, + { + "artist": "Chang Vance", + "song": "Aureolin Dodger", + "id": "29f5dea2-2bd1-4d13-bf33-7aadc4f14ef2", + "priceUsdCents": 324, + "publisher": "DIGIAL" + }, + { + "artist": "May Tanner", + "song": "Cadmium Cultured", + "id": "483b1787-a9fd-4ec8-bf16-5ec122bb559a", + "priceUsdCents": 146, + "publisher": "CHORIZON" + }, + { + "artist": "Tillman Watkins", + "song": "Cameo Dark", + "id": "18f42c1e-8a91-47f9-829e-836ee1a49a17", + "priceUsdCents": 173, + "publisher": "OTHERWAY" + }, + { + "artist": "Naomi Petersen", + "song": "Candy Earth", + "id": "0ed05345-3dfa-4ea4-a59a-6c015153c9d5", + "priceUsdCents": 249, + "publisher": "SONIQUE" + }, + { + "artist": "Kristin Olson", + "song": "Banana Eerie", + "id": "f4ea8c5a-1a5b-4d4b-94cf-74da68c543e6", + "priceUsdCents": 246, + "publisher": "AMTAP" + }, + { + "artist": "Marta Frost", + "song": "Apple Chinese", + "id": "5cd6c545-655c-491d-bc7c-1aaa8569b26c", + "priceUsdCents": 181, + "publisher": "ZAGGLES" + }, + { + "artist": "Etta Anderson", + "song": "Antique English", + "id": "ba865a03-8666-4074-8e47-15bcc707fa6e", + "priceUsdCents": 143, + "publisher": "DATACATOR" + }, + { + "artist": "Charlotte Cooley", + "song": "Amaranth Dim", + "id": "fb4aad81-24c0-4d79-ac34-264923e99464", + "priceUsdCents": 121, + "publisher": "KEENGEN" + }, + { + "artist": "Elise Pugh", + "song": "Bright Cerulean", + "id": "5ac17e68-e6c3-43f6-a305-5c63ac39307f", + "priceUsdCents": 389, + "publisher": "CUJO" + }, + { + "artist": "Tamika Austin", + "song": "Camel Flickr", + "id": "a89ac751-89ba-41e9-9e86-60605b3139f4", + "priceUsdCents": 392, + "publisher": "TECHMANIA" + }, + { + "artist": "Claire Sharp", + "song": "Beaver Electric", + "id": "f9ab6076-3582-479e-877e-f27d5c0919d9", + "priceUsdCents": 209, + "publisher": "PORTICO" + }, + { + "artist": "Lilian Horton", + "song": "Bright Coral", + "id": "634d096f-edc1-4452-a73c-d417c7e46c04", + "priceUsdCents": 227, + "publisher": "GUSHKOOL" + }, + { + "artist": "Reba George", + "song": "Bone Coquelicot", + "id": "a0d5a2c1-7086-46a5-98ea-36724a296728", + "priceUsdCents": 124, + "publisher": "MANGLO" + }, + { + "artist": "Ruiz Vaughn", + "song": "Byzantium Feldgrau", + "id": "202cb417-0b2b-4809-8993-0916aa62ff67", + "priceUsdCents": 223, + "publisher": "RONBERT" + }, + { + "artist": "Wilcox Contreras", + "song": "Brink Cocoa", + "id": "c7763312-b732-4ad9-8d58-295f36e3f411", + "priceUsdCents": 214, + "publisher": "ZENTIA" + }, + { + "artist": "Jodi Ruiz", + "song": "Auburn Chartreuse", + "id": "ab89362d-3ef7-461e-a093-60aa2e28eb07", + "priceUsdCents": 215, + "publisher": "ZOLAVO" + }, + { + "artist": "Potter Keith", + "song": "Arctic CG", + "id": "e52cb1f0-2459-4c60-9841-4497fcdab5a9", + "priceUsdCents": 173, + "publisher": "CIRCUM" + }, + { + "artist": "Wilson Randall", + "song": "Black Coral", + "id": "6726183b-0b9a-4060-a1c6-6dec1e726070", + "priceUsdCents": 206, + "publisher": "GEEKUS" + }, + { + "artist": "Morrow Camacho", + "song": "Bittersweet Poetic", + "id": "23edaed1-d443-4e53-bbbd-c54dc8d3bf5f", + "priceUsdCents": 182, + "publisher": "MEDESIGN" + }, + { + "artist": "Clayton Sears", + "song": "Bitter Dodger", + "id": "17afec57-4fcc-46c0-a63d-04c1ff60f9a2", + "priceUsdCents": 275, + "publisher": "ZOXY" + }, + { + "artist": "Nicholson Pate", + "song": "Cedar CG", + "id": "af78e70f-67c3-4582-80a5-3b4ae04d4695", + "priceUsdCents": 193, + "publisher": "MYOPIUM" + }, + { + "artist": "Madeleine Rivers", + "song": "Bisque Congo", + "id": "596626a8-c54b-4d88-9925-0415d4ab589d", + "priceUsdCents": 137, + "publisher": "LINGOAGE" + }, + { + "artist": "Jeannie Benton", + "song": "Caribbean Denim", + "id": "802afc05-3e72-4c51-b234-7e16a63150e2", + "priceUsdCents": 122, + "publisher": "XURBAN" + }, + { + "artist": "Jefferson Burks", + "song": "Buff Copper", + "id": "90ad25a1-a562-4b56-8f0b-aac08e13fd02", + "priceUsdCents": 162, + "publisher": "ZEAM" + }, + { + "artist": "Diann Osborn", + "song": "Carmine Fallow", + "id": "b654e7dd-a907-4fda-8913-d7c12b65b556", + "priceUsdCents": 357, + "publisher": "HIVEDOM" + }, + { + "artist": "Rose Rowe", + "song": "Air Cornsilk", + "id": "c8c124b1-2f03-4f36-aedd-9bee047914e7", + "priceUsdCents": 128, + "publisher": "IDETICA" + }, + { + "artist": "Ilene Mays", + "song": "Blanched Celadon", + "id": "e2c2e0b4-bb86-4737-a9c3-3135c345e0a5", + "priceUsdCents": 243, + "publisher": "FUTURIZE" + }, + { + "artist": "Margarita Dawson", + "song": "Baby Claret", + "id": "fd86fe4f-384e-494a-ae91-f3f2fd49a06d", + "priceUsdCents": 244, + "publisher": "KATAKANA" + }, + { + "artist": "Dominguez Rocha", + "song": "Brink CG", + "id": "64acc7ea-d104-43d2-8845-33c7678af85d", + "priceUsdCents": 100, + "publisher": "SLOGANAUT" + }, + { + "artist": "Edith Petty", + "song": "Bistre Chocolate", + "id": "8b250802-7b4d-4fdd-ae44-d68fbcfef386", + "priceUsdCents": 144, + "publisher": "ZILIDIUM" + }, + { + "artist": "Rutledge Rosario", + "song": "Alabaster Copper", + "id": "b52dc941-f641-4450-9c5f-24ed83593325", + "priceUsdCents": 244, + "publisher": "SULFAX" + }, + { + "artist": "Lillian Zamora", + "song": "Bittersweet Sunday", + "id": "64b6f717-2524-48ae-b582-c6b059f31a93", + "priceUsdCents": 220, + "publisher": "SOLAREN" + }, + { + "artist": "Walter Villarreal", + "song": "Candy Flax", + "id": "d043213f-01fc-4086-83df-e852b6450605", + "priceUsdCents": 297, + "publisher": "RETRACK" + }, + { + "artist": "Christensen Sandoval", + "song": "Buff Cerulean", + "id": "faec7ffb-0df0-4592-9b04-ea04a1ba8620", + "priceUsdCents": 125, + "publisher": "NITRACYR" + }, + { + "artist": "Frances Hall", + "song": "Amber Poetic", + "id": "b0d3f55a-490c-49f3-a8cb-5bb873e14f27", + "priceUsdCents": 209, + "publisher": "ZIZZLE" + }, + { + "artist": "Nadia Case", + "song": "Burnt Electric", + "id": "c87719b9-e166-471f-a0f6-209fca526583", + "priceUsdCents": 299, + "publisher": "TALKOLA" + }, + { + "artist": "Faulkner Casey", + "song": "Cadmium Cinnamon", + "id": "01f63218-1ceb-400c-bbd7-4c33a8733b03", + "priceUsdCents": 324, + "publisher": "UNI" + }, + { + "artist": "Kristen Gregory", + "song": "Brown English", + "id": "9790b013-ae7c-4afb-b166-37a0a012cc5b", + "priceUsdCents": 180, + "publisher": "INTERODEO" + }, + { + "artist": "Hardy Alexander", + "song": "Aqua November", + "id": "464cd8a0-b5f6-43c1-a6d6-b548c13602f3", + "priceUsdCents": 289, + "publisher": "ISOPOP" + }, + { + "artist": "Jerry Floyd", + "song": "Brown Dartmouth", + "id": "6bef9c2e-0610-49fa-b162-3446aa88757e", + "priceUsdCents": 338, + "publisher": "HOMETOWN" + }, + { + "artist": "Nichole Phelps", + "song": "Blue Cultured", + "id": "c206da83-ad46-4655-86a5-aa95a5bb485e", + "priceUsdCents": 397, + "publisher": "XIXAN" + }, + { + "artist": "Marguerite Mcclure", + "song": "Cedar Columbia", + "id": "4e01c867-3084-4ae4-9c8a-5b0750465037", + "priceUsdCents": 207, + "publisher": "TELEQUIET" + }, + { + "artist": "Beach Evans", + "song": "Bright Coffee", + "id": "26d74b66-5bb7-46e6-89cb-b8864b8111bf", + "priceUsdCents": 120, + "publisher": "OHMNET" + }, + { + "artist": "Clarke Peterson", + "song": "Carnelian Falu", + "id": "d708c366-0580-410b-a0d7-1f7c91ecc7c4", + "priceUsdCents": 339, + "publisher": "LUMBREX" + }, + { + "artist": "Bertie Michael", + "song": "Ash Chinese", + "id": "d25ff9f9-25ae-459d-8591-7650106a3d11", + "priceUsdCents": 204, + "publisher": "CRUSTATIA" + }, + { + "artist": "Barr Good", + "song": "Black Earth", + "id": "882c720a-a0ba-4fe1-9e02-eb2640a3677a", + "priceUsdCents": 153, + "publisher": "OCEANICA" + }, + { + "artist": "Rios Sheppard", + "song": "Blanched Davy", + "id": "38ad21d5-679d-4352-bab6-11a03ea167d3", + "priceUsdCents": 222, + "publisher": "BUZZMAKER" + }, + { + "artist": "Natalia Wooten", + "song": "Alloy Charleston", + "id": "107adc93-dcbb-4ffe-9259-ebe67cdfce32", + "priceUsdCents": 273, + "publisher": "GORGANIC" + }, + { + "artist": "Hopkins Levine", + "song": "Blue China", + "id": "5df65ddf-afe7-4514-93c2-93f53885cf71", + "priceUsdCents": 248, + "publisher": "GINKLE" + }, + { + "artist": "Monique Curry", + "song": "Aqua Cyclamen", + "id": "5148a24f-5c13-4861-96d7-ca957af1b16d", + "priceUsdCents": 226, + "publisher": "INTRADISK" + }, + { + "artist": "Kirsten Decker", + "song": "Ash Cinnamon", + "id": "ca90e78f-d4d3-4af3-9b20-254254a4d922", + "priceUsdCents": 193, + "publisher": "COMTOURS" + }, + { + "artist": "DJ DW", + "song": "Battleship Cinnamon", + "id": "ba66eb9d-f8b5-4924-a9e2-e8817e02b03a", + "priceUsdCents": 365, + "publisher": "MUSICMAN INC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Bright Cerulean", + "id": "84024726-d9ba-4dd4-830d-1022f79eca97", + "priceUsdCents": 166, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "DJ DW", + "song": "Black Crimson", + "id": "c79bcf38-d459-4479-a385-a27a9bdb558a", + "priceUsdCents": 370, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "DJ DW", + "song": "Caribbean Citron", + "id": "8643a896-a7a9-4139-95cc-583e1710b2b1", + "priceUsdCents": 360, + "publisher": "MUSICMAN INC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Carnelian Cobalt", + "id": "5e8d8ff7-25a7-41c6-a1fa-ac823fda0161", + "priceUsdCents": 311, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "Professor RattyB", + "song": "Big Cyan", + "id": "209e59fb-68a5-45a1-aebb-f84447229a4e", + "priceUsdCents": 261, + "publisher": "MUSICMAN INC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Bright Electric", + "id": "ced73077-f8c7-40d0-96ee-8d995baa080b", + "priceUsdCents": 273, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "Professor RattyB", + "song": "Blanched Cornsilk", + "id": "88ca6ef9-8cd4-4a52-9c85-f7d7ac81c275", + "priceUsdCents": 192, + "publisher": "MUSICMAN INC" + }, + { + "artist": "DJ DW", + "song": "Amaranth Fever", + "id": "00ea572d-a3ea-41dc-97f6-50af8b26cd13", + "priceUsdCents": 199, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Cadmium Eggplant", + "id": "30af38c6-f32d-4e7b-a571-e8641247b975", + "priceUsdCents": 168, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Bleu Cinnamon", + "id": "f90cef82-bd0a-44a9-bd3a-bd743c140ced", + "priceUsdCents": 153, + "publisher": "GLOBOMANTICS MUSIC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Carrot Eton", + "id": "dbea9bd8-fe1f-478a-a98a-5b46d481cf57", + "priceUsdCents": 161, + "publisher": "MUSICMAN INC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Baker Firebrick", + "id": "1a4e5bc5-4fa3-4b37-9d36-e15dc9ab6b21", + "priceUsdCents": 312, + "publisher": "MUSICMAN INC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Cadet Celadon", + "id": "fd7667cb-3a41-4777-93bb-ed2d0d8d7458", + "priceUsdCents": 142, + "publisher": "MUSICMAN INC" + }, + { + "artist": "DJ DW", + "song": "Byzantine Charcoal", + "id": "5ab5351f-212a-4fcc-9a76-b5bfbc5d18a0", + "priceUsdCents": 174, + "publisher": "MUSICMAN INC" + }, + { + "artist": "Arturus Ardvarkian", + "song": "Brick Cerulean", + "id": "65ce7700-c32d-415c-9bf4-26f4e4911372", + "priceUsdCents": 381, + "publisher": "GLOBOMANTICS MUSIC" + } + ] \ No newline at end of file diff --git a/boto3/dynamodb/loadData.py b/boto3/dynamodb/loadData.py new file mode 100644 index 0000000..4ee9489 --- /dev/null +++ b/boto3/dynamodb/loadData.py @@ -0,0 +1,26 @@ +import boto3 +import json + +dynamodb = boto3.client('dynamodb') + +def upload(): + with open('data.json', 'r') as datafile: + records = json.load(datafile) + for song in records: + print(song) + item = { + 'artist':{'S':song['artist']}, + 'song':{'S':song['song']}, + 'id':{'S': song['id']}, + 'priceUsdCents':{'S': str(song['priceUsdCents'])}, + 'publisher':{'S': song['publisher']} + } + print(item) + response = dynamodb.put_item( + TableName='basicSongsTable', + Item=item + ) + print("UPLOADING ITEM") + print(response) + +upload() \ No newline at end of file diff --git a/boto3/dynamodb/query.py b/boto3/dynamodb/query.py new file mode 100644 index 0000000..9711572 --- /dev/null +++ b/boto3/dynamodb/query.py @@ -0,0 +1,26 @@ +import boto3 +import json +from boto3.dynamodb.conditions import Key + +TABLE_NAME = "basicSongsTable" + +# Creating the DynamoDB Client +dynamodb_client = boto3.client('dynamodb', region_name="us-west-2") + +# Creating the DynamoDB Table Resource +dynamodb = boto3.resource('dynamodb', region_name="us-west-2") +table = dynamodb.Table(TABLE_NAME) + + +artists = table.scan(AttributesToGet=['artist.']) + + +specific_artist = table.query( + KeyConditionExpression=Key('artist').eq('Arturus Ardvarkian') +) + +artists_object = json.dumps(artists, indent = 4) +print(artists_object) + +specific_artist = json.dumps(specific_artist, indent = 4) +print(specific_artist) \ No newline at end of file diff --git a/boto3/create-instance.py b/boto3/ec2/create-instance.py similarity index 100% rename from boto3/create-instance.py rename to boto3/ec2/create-instance.py diff --git a/generic/exception-handling.py b/generic/exception-handling.py index dc6520a..e69de29 100644 --- a/generic/exception-handling.py +++ b/generic/exception-handling.py @@ -1,19 +0,0 @@ -import os -import sys - -directory = "demodir_" -symbols = "~!@#$%^*()_-+={}[]:>;', Date: Tue, 20 Dec 2022 18:19:43 +0530 Subject: [PATCH 02/10] Update --- boto3/dynamodb/{README.json => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename boto3/dynamodb/{README.json => README.md} (100%) diff --git a/boto3/dynamodb/README.json b/boto3/dynamodb/README.md similarity index 100% rename from boto3/dynamodb/README.json rename to boto3/dynamodb/README.md From 3d437a03db54c13dc36e59e5602ead75d61724b4 Mon Sep 17 00:00:00 2001 From: dv-sharma <65087388+dv-sharma@users.noreply.github.com> Date: Sat, 24 Feb 2024 00:23:04 -0500 Subject: [PATCH 03/10] Create awsrdsmysqlp.py Created a python script to execute query on mysql RDS instance on AWS using boto3,mysql.connector and tabulate module --- awsrdsmysqlp.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 awsrdsmysqlp.py diff --git a/awsrdsmysqlp.py b/awsrdsmysqlp.py new file mode 100644 index 0000000..b342bb2 --- /dev/null +++ b/awsrdsmysqlp.py @@ -0,0 +1,52 @@ +import boto3 +import json +import mysql.connector +import tabulate + +# Initialize AWS Secrets Manager client +secrets_manager = boto3.client('secretsmanager') +db_host = "DB_HOST" + +def get_secretvalue(): + secret_name = secrets_manager.get_secret_value(SecretId='YOUR_SECRET_ARN') + secret_dict = json.loads(secret_name['SecretString']) + db_username = secret_dict['username'] + db_password = secret_dict['password'] + return db_username, db_password + +def execute_query(db_name, query): + db_username, db_password = get_secretvalue() + + for db in db_name: + try: + connection = mysql.connector.connect( + host=db_host, + user=db_username, + password=db_password, + database=db + ) + cursor = connection.cursor() + cursor.execute(query) + result = cursor.fetchall() + row_count = cursor.rowcount + + if query.upper().startswith(("SELECT", "SHOW", "DESCRIBE")): + if result: + header = [column_name[0] for column_name in cursor.description] + table_output = [header] + list(result) + print(f"Running query on: {db}") + print(tabulate.tabulate(table_output, tablefmt="pipe")) + else: + print("Empty result") + else: + print(f"{row_count} rows affected in {db}") + + except Exception as e: + print("Error:", e) + finally: + if 'connection' in locals(): + connection.close() + +db_names = input("Enter the DB names followed by space:").split() +query = input("Enter the query:") +execute_query(db_names, query) From d97c2f25136cbc0c78a833ce1fefba5ad5e3c026 Mon Sep 17 00:00:00 2001 From: dv-sharma <65087388+dv-sharma@users.noreply.github.com> Date: Sat, 24 Feb 2024 02:11:05 -0500 Subject: [PATCH 04/10] Update awsrdsmysqlp.py connection.commit() added to retain INSERT,UPDATE changes --- awsrdsmysqlp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awsrdsmysqlp.py b/awsrdsmysqlp.py index b342bb2..d6e76fd 100644 --- a/awsrdsmysqlp.py +++ b/awsrdsmysqlp.py @@ -39,6 +39,7 @@ def execute_query(db_name, query): else: print("Empty result") else: + connection.commit() print(f"{row_count} rows affected in {db}") except Exception as e: From 1b7fbb54fff5879db9eb26dbe2056f1972807514 Mon Sep 17 00:00:00 2001 From: TechiesCamp <106984297+techiescamp@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:12:50 +0530 Subject: [PATCH 05/10] Update README.md --- README.md | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ff8606f..94e38ec 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Python for DevOps + +# Python for DevOps: 8-Week Learning Roadmap If you want to understand the importance of python for DevOps, please read the [detailed python for DevOps guide](https://devopscube.com/python-for-devops/). @@ -9,3 +10,149 @@ If you are looking for a guided way to learn Python from scratch., I recommend t 1. [learnpython.org](https://www.learnpython.org/) 2. [Learn Python 3 from Scratch](https://www.educative.io/courses/learn-python-3-from-scratch?aff=KNLz) 3. [Python for Beginners – Full Video Course](https://www.youtube.com/watch?v=eWRfhZUzrAc) + + +## πŸ“Œ Overview + +This roadmap is designed to help DevOps Engineers and Platform Engineers master Python for automation, cloud operations, infrastructure as code, and Kubernetes. The curriculum includes hands-on projects that mimic real-world DevOps tasks using industry tools like Docker, Terraform, Kubernetes, and CI/CD pipelines. + +By the end of this roadmap, you'll: +- Automate repetitive tasks with Python +- Work with cloud APIs and infrastructure automation +- Build Python-based DevOps tools and microservices +- Deploy containerized apps with Docker and Kubernetes +- Use AI-powered tools like LlamaIndex & GenAI for DevOps +- Be fully prepared for Python-based DevOps interviews + +--- + +## πŸš€ **Week 1: Python Fundamentals for DevOps** + +### 🎯 **Goal:** Learn the basics of Python for scripting and automation. + +βœ… Python syntax, variables, loops, functions, and error handling +βœ… Working with files (reading, writing, and parsing logs) +βœ… Interacting with OS processes (`subprocess`, `os` modules) +βœ… Installing and managing packages with `pip` and `venv` + +**πŸ”¨ Hands-on Project:** +- Write a Python script to automate log parsing from `/var/log/` and extract useful insights. +- Create a script that monitors CPU & memory usage using the `psutil` library. + +--- + +## πŸ›  **Week 2: Automating DevOps Tasks with Python** + +### 🎯 **Goal:** Automate daily DevOps operations. + +βœ… Working with APIs (`requests` library for REST APIs) +βœ… Automating SSH tasks (`paramiko` for remote execution) +βœ… Automating cloud operations with AWS/GCP SDKs (`boto3` and `google-cloud-sdk`) +βœ… Writing CLI tools with `argparse` + +**πŸ”¨ Hands-on Project:** +- Build a Python script that automates AWS EC2 instance management (start, stop, terminate instances). +- Create a CLI tool that checks the status of Kubernetes pods using `kubectl` and `subprocess`. + +--- + +## 🐳 **Week 3: Docker & Python for Containerized Applications** + +### 🎯 **Goal:** Learn to containerize Python applications for DevOps automation. + +βœ… Writing Dockerfiles for Python apps +βœ… Running Python scripts inside containers +βœ… Docker Compose for multi-container apps +βœ… Working with Python SDK for Docker (`docker-py`) + +**πŸ”¨ Hands-on Project:** +- Build a Python script to manage Docker containers (start, stop, restart, delete containers). +- Create a Flask API that returns system metrics (CPU, RAM) and deploy it in a Docker container. + +--- + +## πŸ”§ **Week 4: Infrastructure as Code (IaC) with Python** + +### 🎯 **Goal:** Automate infrastructure provisioning with Python. + +βœ… Terraform automation with Python (`python-terraform`) +βœ… Working with Ansible and Python (`ansible-runner`) +βœ… Writing Python scripts to automate Kubernetes YAML generation +βœ… Using `Fabric` for remote automation + +**πŸ”¨ Hands-on Project:** +- Write a Python script that provisions AWS infrastructure (VPC, EC2, S3) using Terraform. +- Automate Ansible playbook execution using Python. + +--- + +## ☸️ **Week 5: Kubernetes Automation with Python** + +### 🎯 **Goal:** Automate Kubernetes operations using Python. + +βœ… Working with Kubernetes Python SDK (`kubernetes` library) +βœ… Managing Kubernetes objects dynamically with Python +βœ… Writing Admission Webhooks in Python +βœ… Automating Helm deployments with Python + +**πŸ”¨ Hands-on Project:** +- Write a Python script that dynamically creates and deletes Kubernetes namespaces. +- Build a **Mutating Admission Webhook** to enforce security policies in a cluster. + +--- + +## πŸ” **Week 6: Python for Security & Monitoring in DevOps** + +### 🎯 **Goal:** Secure infrastructure and monitor logs with Python. + +βœ… Parsing and analyzing logs (`loguru`, `logging`) +βœ… Security automation (checking misconfigurations with `PyInfra`) +βœ… Python for **SIEM (Security Information & Event Management)** +βœ… Automating RBAC checks for Kubernetes + +**πŸ”¨ Hands-on Project:** +- Build a tool that checks Kubernetes RBAC permissions and finds over-privileged service accounts. +- Automate security scanning of container images using Trivy and Python. + +--- + +## πŸ€– **Week 7: GenAI & LlamaIndex for DevOps** + +### 🎯 **Goal:** Use **AI for DevOps** workflows with Python. + +βœ… Introduction to **LlamaIndex & GenAI for DevOps** +βœ… Automating incident response with AI-driven bots +βœ… Generating YAML/JSON configurations using AI +βœ… AI-powered log analysis using `LangChain` & `OpenAI API` + +**πŸ”¨ Hands-on Project:** +- Build an AI-powered chatbot that suggests **Kubernetes troubleshooting steps**. +- Develop an **AI-based log anomaly detection system** that detects security threats. + +--- + +## πŸ€– **Week 8: Agentic AI & Final Capstone Projects** + +### 🎯 **Goal:** Implement **advanced AI-driven DevOps automation**. + +βœ… Understanding **Agentic AI** for DevOps +βœ… AI-driven **CI/CD pipeline optimization** +βœ… Automating incident remediation using AI agents +βœ… Integrating AI with monitoring tools (Prometheus, Grafana) + +### **Final Capstone Projects** +1️⃣ **DevOps Dashboard API:** Build a **FastAPI** app that displays **real-time Kubernetes metrics**. +2️⃣ **Self-Healing Kubernetes System:** AI-powered Kubernetes operator that auto-heals pods based on anomaly detection. +3️⃣ **Intelligent CI/CD Analyzer:** AI-driven **CI/CD log analyzer** that suggests fixes based on failure patterns. + +--- + +## πŸ“’ **Next Steps** + +- Contribute to Open Source Python DevOps projects +- Deploy Python automation scripts in production Kubernetes environments +- Prepare for DevOps interviews with Python scripting questions +- Build AI-powered DevOps tools using LangChain & LlamaIndex + +πŸ”₯ Ready to Master Python for DevOps? Let’s Get Started! πŸš€ + From 3dd5061921376b9711c77771c877e11be326a979 Mon Sep 17 00:00:00 2001 From: TechiesCamp <106984297+techiescamp@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:16:25 +0530 Subject: [PATCH 06/10] Update README.md --- README.md | 71 +++++++++++++++++++++++-------------------------------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 94e38ec..8b91b8b 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,6 @@ If you want to understand the importance of python for DevOps, please read the [detailed python for DevOps guide](https://devopscube.com/python-for-devops/). -## Python Learning Resources - -If you are looking for a guided way to learn Python from scratch., I recommend the following resources. - -1. [learnpython.org](https://www.learnpython.org/) -2. [Learn Python 3 from Scratch](https://www.educative.io/courses/learn-python-3-from-scratch?aff=KNLz) -3. [Python for Beginners – Full Video Course](https://www.youtube.com/watch?v=eWRfhZUzrAc) - - ## πŸ“Œ Overview This roadmap is designed to help DevOps Engineers and Platform Engineers master Python for automation, cloud operations, infrastructure as code, and Kubernetes. The curriculum includes hands-on projects that mimic real-world DevOps tasks using industry tools like Docker, Terraform, Kubernetes, and CI/CD pipelines. @@ -24,11 +15,17 @@ By the end of this roadmap, you'll: - Use AI-powered tools like LlamaIndex & GenAI for DevOps - Be fully prepared for Python-based DevOps interviews ---- +## πŸ‘¨β€πŸ’» Python Learning Resources -## πŸš€ **Week 1: Python Fundamentals for DevOps** +If you are looking for a guided way to learn Python from scratch., I recommend the following resources. -### 🎯 **Goal:** Learn the basics of Python for scripting and automation. +1. [learnpython.org](https://www.learnpython.org/) +2. [Learn Python 3 from Scratch](https://www.educative.io/courses/learn-python-3-from-scratch?aff=KNLz) +3. [Python for Beginners – Full Video Course](https://www.youtube.com/watch?v=eWRfhZUzrAc) + +## πŸš€ Week 1: Python Fundamentals for DevOps + +### 🎯 Goal: Learn the basics of Python for scripting and automation. βœ… Python syntax, variables, loops, functions, and error handling βœ… Working with files (reading, writing, and parsing logs) @@ -39,11 +36,9 @@ By the end of this roadmap, you'll: - Write a Python script to automate log parsing from `/var/log/` and extract useful insights. - Create a script that monitors CPU & memory usage using the `psutil` library. ---- - -## πŸ›  **Week 2: Automating DevOps Tasks with Python** +## πŸ›  Week 2: Automating DevOps Tasks with Python -### 🎯 **Goal:** Automate daily DevOps operations. +### 🎯 Goal: Automate daily DevOps operations. βœ… Working with APIs (`requests` library for REST APIs) βœ… Automating SSH tasks (`paramiko` for remote execution) @@ -54,11 +49,9 @@ By the end of this roadmap, you'll: - Build a Python script that automates AWS EC2 instance management (start, stop, terminate instances). - Create a CLI tool that checks the status of Kubernetes pods using `kubectl` and `subprocess`. ---- - -## 🐳 **Week 3: Docker & Python for Containerized Applications** +## 🐳 Week 3: Docker & Python for Containerized Applications -### 🎯 **Goal:** Learn to containerize Python applications for DevOps automation. +### 🎯 Goal:** Learn to containerize Python applications for DevOps automation. βœ… Writing Dockerfiles for Python apps βœ… Running Python scripts inside containers @@ -69,11 +62,9 @@ By the end of this roadmap, you'll: - Build a Python script to manage Docker containers (start, stop, restart, delete containers). - Create a Flask API that returns system metrics (CPU, RAM) and deploy it in a Docker container. ---- - -## πŸ”§ **Week 4: Infrastructure as Code (IaC) with Python** +## πŸ”§ Week 4: Infrastructure as Code (IaC) with Python -### 🎯 **Goal:** Automate infrastructure provisioning with Python. +### 🎯 Goal: Automate infrastructure provisioning with Python. βœ… Terraform automation with Python (`python-terraform`) βœ… Working with Ansible and Python (`ansible-runner`) @@ -86,9 +77,9 @@ By the end of this roadmap, you'll: --- -## ☸️ **Week 5: Kubernetes Automation with Python** +## ☸️ Week 5: Kubernetes Automation with Python -### 🎯 **Goal:** Automate Kubernetes operations using Python. +### 🎯 Goal: Automate Kubernetes operations using Python. βœ… Working with Kubernetes Python SDK (`kubernetes` library) βœ… Managing Kubernetes objects dynamically with Python @@ -99,11 +90,9 @@ By the end of this roadmap, you'll: - Write a Python script that dynamically creates and deletes Kubernetes namespaces. - Build a **Mutating Admission Webhook** to enforce security policies in a cluster. ---- - -## πŸ” **Week 6: Python for Security & Monitoring in DevOps** +## πŸ” Week 6: Python for Security & Monitoring in DevOps. -### 🎯 **Goal:** Secure infrastructure and monitor logs with Python. +### 🎯 Goal: Secure infrastructure and monitor logs with Python. βœ… Parsing and analyzing logs (`loguru`, `logging`) βœ… Security automation (checking misconfigurations with `PyInfra`) @@ -114,11 +103,9 @@ By the end of this roadmap, you'll: - Build a tool that checks Kubernetes RBAC permissions and finds over-privileged service accounts. - Automate security scanning of container images using Trivy and Python. ---- - -## πŸ€– **Week 7: GenAI & LlamaIndex for DevOps** +## πŸ€– Week 7: GenAI & LlamaIndex for DevOps -### 🎯 **Goal:** Use **AI for DevOps** workflows with Python. +### 🎯 Goal: Use AI for DevOps workflows with Python. βœ… Introduction to **LlamaIndex & GenAI for DevOps** βœ… Automating incident response with AI-driven bots @@ -131,23 +118,23 @@ By the end of this roadmap, you'll: --- -## πŸ€– **Week 8: Agentic AI & Final Capstone Projects** +## πŸ€– Week 8: Agentic AI & Final Capstone Projects -### 🎯 **Goal:** Implement **advanced AI-driven DevOps automation**. +### 🎯 Goal: Implement **advanced AI-driven DevOps automation -βœ… Understanding **Agentic AI** for DevOps -βœ… AI-driven **CI/CD pipeline optimization** +βœ… Understanding Agentic AI for DevOps +βœ… AI-driven CI/CD pipeline optimization βœ… Automating incident remediation using AI agents βœ… Integrating AI with monitoring tools (Prometheus, Grafana) -### **Final Capstone Projects** -1️⃣ **DevOps Dashboard API:** Build a **FastAPI** app that displays **real-time Kubernetes metrics**. +### Final Capstone Projects +1️⃣ **DevOps Dashboard API:** Build a FastAPI app that displays real-time Kubernetes metrics. 2️⃣ **Self-Healing Kubernetes System:** AI-powered Kubernetes operator that auto-heals pods based on anomaly detection. -3️⃣ **Intelligent CI/CD Analyzer:** AI-driven **CI/CD log analyzer** that suggests fixes based on failure patterns. +3️⃣ **Intelligent CI/CD Analyzer:** AI-driven CI/CD log analyzer that suggests fixes based on failure patterns. --- -## πŸ“’ **Next Steps** +## πŸ“’ Next Steps - Contribute to Open Source Python DevOps projects - Deploy Python automation scripts in production Kubernetes environments From 1117f7264d4772f8bc91a17732f4e7919d95c7c3 Mon Sep 17 00:00:00 2001 From: TechiesCamp <106984297+techiescamp@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:17:34 +0530 Subject: [PATCH 07/10] Update README.md --- README.md | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8b91b8b..5913649 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ If you are looking for a guided way to learn Python from scratch., I recommend t ## 🐳 Week 3: Docker & Python for Containerized Applications -### 🎯 Goal:** Learn to containerize Python applications for DevOps automation. +### 🎯 Goal: Learn to containerize Python applications for DevOps automation. βœ… Writing Dockerfiles for Python apps βœ… Running Python scripts inside containers @@ -75,8 +75,6 @@ If you are looking for a guided way to learn Python from scratch., I recommend t - Write a Python script that provisions AWS infrastructure (VPC, EC2, S3) using Terraform. - Automate Ansible playbook execution using Python. ---- - ## ☸️ Week 5: Kubernetes Automation with Python ### 🎯 Goal: Automate Kubernetes operations using Python. @@ -88,7 +86,7 @@ If you are looking for a guided way to learn Python from scratch., I recommend t **πŸ”¨ Hands-on Project:** - Write a Python script that dynamically creates and deletes Kubernetes namespaces. -- Build a **Mutating Admission Webhook** to enforce security policies in a cluster. +- Build a Mutating Admission Webhook to enforce security policies in a cluster. ## πŸ” Week 6: Python for Security & Monitoring in DevOps. @@ -113,14 +111,13 @@ If you are looking for a guided way to learn Python from scratch., I recommend t βœ… AI-powered log analysis using `LangChain` & `OpenAI API` **πŸ”¨ Hands-on Project:** -- Build an AI-powered chatbot that suggests **Kubernetes troubleshooting steps**. -- Develop an **AI-based log anomaly detection system** that detects security threats. +- Build an AI-powered chatbot that suggests Kubernetes troubleshooting steps. +- Develop an AI-based log anomaly detection system that detects security threats. ---- ## πŸ€– Week 8: Agentic AI & Final Capstone Projects -### 🎯 Goal: Implement **advanced AI-driven DevOps automation +### 🎯 Goal: Implement advanced AI-driven DevOps automation βœ… Understanding Agentic AI for DevOps βœ… AI-driven CI/CD pipeline optimization @@ -132,8 +129,6 @@ If you are looking for a guided way to learn Python from scratch., I recommend t 2️⃣ **Self-Healing Kubernetes System:** AI-powered Kubernetes operator that auto-heals pods based on anomaly detection. 3️⃣ **Intelligent CI/CD Analyzer:** AI-driven CI/CD log analyzer that suggests fixes based on failure patterns. ---- - ## πŸ“’ Next Steps - Contribute to Open Source Python DevOps projects From 581999c100d94799abe1f7faf8d46c932125ecd5 Mon Sep 17 00:00:00 2001 From: TechiesCamp <106984297+techiescamp@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:18:27 +0530 Subject: [PATCH 08/10] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5913649..4023daf 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,8 @@ If you are looking for a guided way to learn Python from scratch., I recommend t βœ… Automating incident remediation using AI agents βœ… Integrating AI with monitoring tools (Prometheus, Grafana) -### Final Capstone Projects +## πŸ’Ό Final Capstone Projects + 1️⃣ **DevOps Dashboard API:** Build a FastAPI app that displays real-time Kubernetes metrics. 2️⃣ **Self-Healing Kubernetes System:** AI-powered Kubernetes operator that auto-heals pods based on anomaly detection. 3️⃣ **Intelligent CI/CD Analyzer:** AI-driven CI/CD log analyzer that suggests fixes based on failure patterns. From 3342d246960c586953609743061136cab146df38 Mon Sep 17 00:00:00 2001 From: TechiesCamp <106984297+techiescamp@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:18:47 +0530 Subject: [PATCH 09/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4023daf..c2d741e 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ If you are looking for a guided way to learn Python from scratch., I recommend t - Develop an AI-based log anomaly detection system that detects security threats. -## πŸ€– Week 8: Agentic AI & Final Capstone Projects +## πŸ€– Week 8: Agentic AI ### 🎯 Goal: Implement advanced AI-driven DevOps automation From 7e1742bf1d2363b191529ec433f519b2ada211e6 Mon Sep 17 00:00:00 2001 From: TechiesCamp Date: Tue, 8 Jul 2025 09:26:05 +0530 Subject: [PATCH 10/10] [Add] Azure OpenAPI Python example --- azure-openapi/basic-api/main.py | 32 ++++++++++++++++++++++++ azure-openapi/basic-api/requirements.txt | 1 + 2 files changed, 33 insertions(+) create mode 100644 azure-openapi/basic-api/main.py create mode 100644 azure-openapi/basic-api/requirements.txt diff --git a/azure-openapi/basic-api/main.py b/azure-openapi/basic-api/main.py new file mode 100644 index 0000000..f1f7cf6 --- /dev/null +++ b/azure-openapi/basic-api/main.py @@ -0,0 +1,32 @@ +import os +from openai import AzureOpenAI + +endpoint = "" +model_name = "o4-mini" +deployment = "o4-mini" + +subscription_key = os.getenv("AZURE_OPENAI_API_KEY") +api_version = "2024-12-01-preview" + +client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=subscription_key, +) + +response = client.chat.completions.create( + messages=[ + { + "role": "system", + "content": "You are a senior DevOps engineer helping beginners.", + }, + { + "role": "user", + "content": "How do I set up CI/CD using GitHub Actions for a Node.js app?", + } + ], + max_completion_tokens=100000, + model=deployment +) + +print(response.choices[0].message.content) \ No newline at end of file diff --git a/azure-openapi/basic-api/requirements.txt b/azure-openapi/basic-api/requirements.txt new file mode 100644 index 0000000..c9a4ecb --- /dev/null +++ b/azure-openapi/basic-api/requirements.txt @@ -0,0 +1 @@ +openai~=1.60.2