You are viewing our older product's guide. Click here for the documentation of GoodData Cloud, our latest and most advanced product.

Types of Input Data Sources

The data that the bricks use as input for further processing can be stored in different locations.

The input_source parameter of a brick points to the location from which the input data should be obtained. This parameter accepts the location described by a JSON structure. For information about how to enter parameters represented by JSON, see Specifying Complex Parameters.

Azure Blob Storage

The expected data format is a CSV or Parquet file.

Be sure to enter the connectionString parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "blobStorage_client": {
    "connectionString": "{your_connection_string}",
    "container": "{container_name}",
    "path": "{path/to/source/file}"
  },
  "input_source": {
    "type": "blobStorage",
    "file": "{file_name}"
  }
}

Example:

{
  "blobStorage_client": {
    "connectionString": "${blobStorage_client_connection_string}",
    "container": "acme",
    "path": "folder/upload"
  },
  "input_source": {
    "type": "blobStorage",
    "file": "users.csv"
  }
}

Azure SQL Database

The expected data format is a query to your Azure SQL Database.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

Basic authentication:

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://{database_address}:{port}",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT {some_data} FROM {schema_name}.{some_table}"
  }
}

Azure Active Directory authentication:

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://{database_address}:{port}",
      "authentication": {
        "activeDirectoryPassword": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT {some_data} FROM {schema_name}.{some_table}"
  }
}

Example (for basic authentication):

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://acme_gdc.com:1433",
      "authentication": {
        "basic": {
          "userName": "john.doe@gooddata.com",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "sslMode": "verify-full"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT * FROM ANALYTICS.users"
  }
}

Azure Synapse Analytics

The expected data format is a query to your Azure Synapse Analytics.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

Basic authentication:

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://{database_address}:{port}",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT {some_data} FROM {schema_name}.{some_table}"
  }
}

Azure Active Directory authentication:

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://{database_address}:{port}",
      "authentication": {
        "activeDirectoryPassword": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT {some_data} FROM {schema_name}.{some_table}"
  }
}

Example (for basic authentication):

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://acme_gdc.com:1433",
      "authentication": {
        "basic": {
          "userName": "john.doe@gooddata.com",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "sslMode": "verify-full"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT * FROM ANALYTICS.users"
  }
}

BigQuery

The expected data format is a query to your BigQuery data warehouse.

Be sure to enter the private_key parameter as a secure parameter as described in Specifying Complex Parameters.

{  
  "bigquery_client": {
    "connection": {
      "authentication": {
        "serviceAccount": {
          "clientEmail": "{your_username@domain.com}",
          "privateKey": "{your_private_key}"
        }
      },
      "schema": "{schema_name}",
      "project": "{project_name}"
    }    
  },
  "input_source": {
    "type": "bigquery",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example:

{  
  "bigquery_client": {
    "connection": {
      "authentication": {
        "serviceAccount": {
          "clientEmail": "john.doe@gooddata.com",
          "privateKey": "${serviceAccount_private_key}"
        }
      },
      "schema": "ANALYTICS",
      "project": "ACME"
    }    
  },
  "input_source": {
    "type": "bigquery",
    "query": "SELECT * FROM users"
  }
}

GoodData Data Warehouse (ADS)

The expected data format is a query to an ADS instance.

For more information, see Data Warehouse.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "ads_client": {
    "username": "{your_username@domain.com}",
    "password": "{your_password}",
    "ads_id": "{ads_instance_id}"
  },
  "input_source": {
    "type": "ads",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

If your domain is white-labeled, replace ads_id with jdbc_url:

{
  "ads_client": {
    "username": "{your_username@domain.com}",
    "password": "{your_password}",
    "jdbc_url": "jdbc:gdc:datawarehouse://{your.domain.com}/gdc/datawarehouse/instances/{ads_instance_id}"
  },
  "input_source": {
    "type": "ads",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example (for not white-labeled domains):

{
  "ads_client": {
    "username": "john.doe@gooddata.com",
    "password": "${ads_client_password}",
    "ads_id": "123456abcdef7890"
  },
  "input_source": {
    "type": "ads",
    "query": "SELECT * FROM users"
  }
}

Example (for white-labeled domains):

{
  "ads_client": {
    "username": "john.doe@gooddata.com",
    "password": "${ads_client_password}",
    "jdbc_url": "jdbc:gdc:datawarehouse://my.company.com/gdc/datawarehouse/instances/123456abcdef7890"
  },
  "input_source": {
    "type": "ads",
    "query": "SELECT * FROM users"
  }
}

Microsoft SQL Server

The expected data format is a query to your Microsoft SQL Server.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://{database_address}:{port}",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT {some_data} FROM {schema_name}.{some_table}"
  }
}

Example:

{
  "mssql_client": {
    "connection": {
      "url": "jdbc:sqlserver://acme_gdc.com:1433",
      "authentication": {
        "basic": {
          "userName": "john.doe@gooddata.com",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "sslMode": "verify-full"
    }
  },
  "input_source": {
    "type": "mssql",
    "query": "SELECT * FROM ANALYTICS.users"
  }
}

MongoDB Connector for BI

The expected data format is a query to your MongoDB Connector for BI.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "mysql_client": {
    "connection": {
      "url": "jdbc:mysql://{database_address}:{port}",
      "database": "{database_name}",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "sslMode": "{prefer|require|verify-full}",
      "databaseType": "MongoDBConnector",
    }
  },
  "input_source": {
    "type": "mysql",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example:

{
  "mysql_client": {
    "connection": {
      "url": "jdbc:mysql://acme_gdc.com:3306",
      "database": "ACME",
      "authentication": {
        "basic": {
          "userName": "john.doe@gooddata.com",
          "password": "${basic_password}"
        }
      },
      "sslMode": "verify-full",
      "databaseType": "MongoDBConnector",
     }
  },
  "input_source": {
    "type": "mysql",
    "query": "SELECT * FROM users"
  }
}

MySQL

The expected data format is a query to your MySQL database.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "mysql_client": {
    "connection": {
      "url": "jdbc:mysql://{database_address}:{port}",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "mysql",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example:

{
  "mysql_client": {
    "connection": {
      "url": "jdbc:mysql://acme_gdc.com:3306",
      "authentication": {
        "basic": {
          "userName": "john.doe@gooddata.com",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "sslMode": "verify-full"
    }
  },
  "input_source": {
    "type": "mysql",
    "query": "SELECT * FROM users"
  }
}

PostgreSQL

The expected data format is a query to your PostgreSQL database.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "postgresql_client": {
    "connection": {
      "url": "jdbc:postgresql://{database_address}:{port}",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "schema": "{schema_name}",
      "sslMode": "{prefer|require|verify-full}"
    }
  },
  "input_source": {
    "type": "postgresql",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example:

{
  "postgresql_client": {
    "connection": {
      "url": "jdbc:postgresql://acme_gdc.com:5432",
      "authentication": {
        "basic": {
          "userName": "GOODDATA_INTEGRATION",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "schema": "ANALYTICS",
      "sslMode": "verify-full"
    }
  },
  "input_source": {
    "type": "postgresql",
    "query": "SELECT * FROM users"
  }
}

Redshift

The expected data format is a query to your Redshift data warehouse.

Be sure to enter the password or secretAccessKey parameter as a secure parameter as described in Specifying Complex Parameters.

Basic authentication:

{
  "redshift_client": {
    "connection": { 
      "url": "jdbc:redshift://{your_account}.amazonaws.com",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "schema": "{schema_name}"
    }
  },
  "input_source": {
    "type": "redshift",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

IAM authentication:

{
  "redshift_client": {
    "connection": { 
      "url": "jdbc:redshift://{your_account}.amazonaws.com",
      "authentication": {
        "iam": {
          "dbUser": "{your_username}",
          "accessKeyId": "{your_access_key}",
          "secretAccessKey": "{your_secret_access_key}"
        }
      },
      "database": "{database_name}",
      "schema": "{schema_name}"
    }
  },
  "input_source": {
    "type": "redshift",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example (for basic authentication):

{
  "redshift_client": {
    "connection": { 
      "url": "jdbc:redshift://examplecluster.us-west-2.redshift.amazonaws.com",
      "authentication": {
        "basic": {
          "userName": "GOODDATA_INTEGRATION",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "schema": "ANALYTICS"
    }
  },
  "input_source": {
    "type": "redshift",
    "query": "SELECT * FROM users"
  }
}

S3

The expected data format is a CSV or Parquet file.

Be sure to enter the secretKey parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "s3_client": {
    "bucket": "{bucket_name}",
    "accessKey": "{your_access_key}",
    "secretKey": "{your_secret_key}",
    "region": "{bucket_region_code}",
    "serverSideEncryption": "true|false"
  },
  "input_source": {
    "type": "s3",
    "file": "{path/to/file_name}”
  }
}

Example:

{
  "s3_client": {
    "bucket": "users_data",
    "accessKey": "123456789",
    "secretKey": "${s3_client_secret_key}",
    "region": "us-east-1",
    "serverSideEncryption": "true"
  },
  "input_source": {
    "type": "s3",
    "file": "folder/users.csv”
  }
}

The following structure of the input_source parameter for an S3 location is deprecated:

{
  "aws_client": {
    "access_key_id": "{your_access_key_ID}",
    "secret_access_key": "{your_secret_access_key}",
    "region": "{bucket_region_code}"
  },
  "input_source": {
    "type": "s3",
    "key": "{/path/to/file_name}",
    "bucket": "{bucket_name}"
  }
}

Snowflake

The expected data format is a query to your Snowflake data warehouse.

Be sure to enter the password parameter as a secure parameter as described in Specifying Complex Parameters.

{
  "snowflake_client": {
    "connection": {
      "url": "jdbc:snowflake://{your_account}.snowflakecomputing.com",
      "authentication": {
        "basic": {
          "userName": "{your_username}",
          "password": "{your_password}"
        }
      },
      "database": "{database_name}",
      "schema": "{schema_name}",
      "warehouse": "{warehouse_name}"
    }
  },
  "input_source": {
    "type": "snowflake",
    "query": "SELECT {some_data} FROM {some_table}"
  }
}

Example:

{
  "snowflake_client": {
    "connection": {
      "url": "jdbc:snowflake://acme.snowflakecomputing.com",
      "authentication": {
        "basic": {
          "userName": "GOODDATA_INTEGRATION",
          "password": "${basic_password}"
        }
      },
      "database": "ACME",
      "schema": "ANALYTICS",
      "warehouse": "GOODDATA_INTEGRATION"
    }
  },
  "input_source": {
    "type": "snowflake",
    "query": "SELECT * FROM users"
  }
}

Web Location

The expected data format is a CSV file.

{
  "input_source": {
    "type": "web",
    "url": "URL/to/file_name.csv"
  }
}

Example:

{
  "input_source": {
    "type": "web",
    "url": "https://files.acme.com/data/upload/users.csv"
  }
}

Workspace-Specific Data Storage

The expected data format is a CSV file.

For more information about the data storage, see Workspace Specific Data Storage.

{
  "input_source": {
    "type": "staging",
    "path": "{file_name}.csv"
  }
}

Example:

{
  "input_source": {
    "type": "staging",
    "path": "users.csv"
  }
}

Specifying Complex Parameters