Access storage account from Azure databricks using Storage account key.

from pyspark.sql import SparkSession

# Create a Spark Session
spark = SparkSession.builder.getOrCreate()

# Define the storage account and container details
storage_account_name = 'jbadbstorageaccount'
storage_account_key = '<<Account_Key>>'
container_name = 'deltatables'

# Define the configuration for the Azure Storage account
spark.conf.set(
    f"fs.azure.account.key.{storage_account_name}.dfs.core.windows.net",
    storage_account_key
)

# Define the path to the CSV file
file_path = f"abfss://{container_name}@{storage_account_name}.dfs.core.windows.net/customer/customers.csv"

# Read the CSV file into a DataFrame
df = spark.read.format('csv').option('header', 'true').load(file_path)

# Display the DataFrame
display(df)

Access storage account from Azure databricks with Access keys placed in Azure key vault

from pyspark.sql import SparkSession

# Create a Spark Session
spark = SparkSession.builder.getOrCreate()

# Define the storage account and container details
storage_account_name = 'jbadbstorageaccount'
container_name = 'deltatables'

# Retrieve the storage account key from Azure Key Vault
key_vault_scope = 'jbswikidatabricksscopekeyvaultintegration'
key_vault_key = 'jbswikidatabrickssecret'

storage_account_key = dbutils.secrets.get(scope=key_vault_scope, key=key_vault_key)

# Define the configuration for the Azure Storage account
spark.conf.set(
    f"fs.azure.account.key.{storage_account_name}.dfs.core.windows.net",
    storage_account_key
)

# Define the path to the CSV file
file_path = f"abfss://{container_name}@{storage_account_name}.dfs.core.windows.net/customer/customers.csv"

# Read the CSV file into a DataFrame
df = spark.read.format('csv').option('header', 'true').load(file_path)

# Display the DataFrame
display(df)

Regards;
Vivek Janakiraman

Disclaimer:
The views expressed on this blog are mine alone and do not reflect the views of my company or anyone else. All postings on this blog are provided “AS IS” with no warranties, and confers no rights.