How to use the carto.sql.CopySQLClient function in carto

To help you get started, we’ve selected a few carto examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github CartoDB / cartoframes / tests / unit / core / managers / test_context_manager.py View on Github external
def test_internal_copy_from(self, mocker):
        # Given
        from shapely.geometry import Point
        mocker.patch('cartoframes.core.managers.context_manager._create_auth_client')
        mock = mocker.patch.object(CopySQLClient, 'copyfrom')
        cdf = CartoDataFrame({'A': [1, 2], 'B': [Point(0, 0), Point(1, 1)]})
        columns = [
            ColumnInfo('A', 'a', 'bigint', False),
            ColumnInfo('B', 'b', 'geometry', True)
        ]

        # When
        cm = ContextManager(self.credentials)
        cm._copy_from(cdf, 'table_name', columns)

        # Then
        assert mock.call_args[0][0] == '''
            COPY table_name(a,b) FROM stdin WITH (FORMAT csv, DELIMITER '|', NULL '__null');
        '''.strip()
        assert list(mock.call_args[0][1]) == [
            b'1|0101000020E610000000000000000000000000000000000000\n',
github CartoDB / carto-python / tests / test_sql_copy.py View on Github external
def copy_client(api_key_auth_client_usr):
    return CopySQLClient(api_key_auth_client_usr)
github CartoDB / carto-python / examples / copy_and_pandas_example.py View on Github external
logger.error('You need to provide valid credentials, run with '
                 '-h parameter for details')
    sys.exit(1)

# Create and cartodbfy a table
sqlClient = SQLClient(auth_client)
sqlClient.send("""
    CREATE TABLE IF NOT EXISTS copy_example (
      the_geom geometry(Geometry,4326),
      name text,
      age integer
    )
    """)
sqlClient.send("SELECT CDB_CartodbfyTable(current_schema, 'copy_example')")

copyClient = CopySQLClient(auth_client)

# COPY FROM example
logger.info("COPY'ing FROM file...")
query = ('COPY copy_example (the_geom, name, age) '
         'FROM stdin WITH (FORMAT csv, HEADER true)')
result = copyClient.copyfrom_file_path(query, 'files/copy_from.csv')
logger.info('result = %s' % result)

# COPY TO example with pandas DataFrame
logger.info("COPY'ing TO pandas DataFrame...")
query = 'COPY copy_example TO stdout WITH (FORMAT csv, HEADER true)'
result = copyClient.copyto_stream(query)
df = pd.read_csv(result)
logger.info(df.head())

# Truncate the table to make this example repeatable
github CartoDB / carto-python / examples / copy_chunked_file.py View on Github external
BATCH_TERMINAL_STATES = ['done', 'failed', 'cancelled', 'unknown']

logger.info("Dropping, re-creating and cartodbfy'ing the table through the Batch API...")
batchSQLClient = BatchSQLClient(auth_client)
job = batchSQLClient.create(LIST_OF_SQL_QUERIES)
while not job['status'] in BATCH_TERMINAL_STATES:
    time.sleep(1)
    job = batchSQLClient.read(job['job_id'])
if job['status'] != 'done':
    logger.error('Could not create and cartodbfy table')
    logger.error(job['failed_reason'])
    sys.exit(1)
logger.info("Table created and cartodbfy'ed")

copyClient = CopySQLClient(auth_client)

# COPY FROM example
logger.info("COPY'ing the table FROM the file...")
query = 'COPY tl_2014_census_tracts (the_geom,statefp,countyfp,tractce,geoid,name,cartodb_id,aland,awater,created_at,updated_at,mtfcc) FROM stdin WITH (FORMAT csv, HEADER true)'
input_file = '/home/rtorre/src/cartodb-tests/sql_copy_perf/tl_2014_census_tracts.csv'

result = copyClient.copyfrom_file_path(query, input_file)

logger.info(result)
logger.info("Table populated")
github CartoDB / cartoframes / cartoframes / lib / context / api_context.py View on Github external
def __init__(self, credentials):
        self.auth_client = APIKeyAuthClient(
            base_url=credentials.base_url,
            api_key=credentials.api_key,
            session=credentials.session,
            client_id='cartoframes_{}'.format(__version__),
            user_agent='cartoframes_{}'.format(__version__)
        )
        self.sql_client = SQLClient(self.auth_client)
        self.copy_client = CopySQLClient(self.auth_client)
        self.batch_sql_client = BatchSQLClient(self.auth_client)
github CartoDB / carto-python / examples / duplicate_table.py View on Github external
# Create the table in the destination account
logger.info('Creating the table in the destination account...')
res = sql_dst_client.send(create_table_no_seqs)
logger.info('Response: {}'.format(res))

# Cartodbfy the table (this is optional)
logger.info("Cartodbfy'ing the destination table...")
res = sql_dst_client.send(
    "SELECT CDB_CartodbfyTable(current_schema, '%s')" % TABLE_NAME
)
logger.info('Response: {}'.format(res))

# Create COPY clients
copy_src_client = CopySQLClient(auth_src_client)
copy_dst_client = CopySQLClient(auth_dst_client)

# COPY (streaming) the data from the source to the dest table. We use
# here all the COPY defaults. Note that we take the `response` from
# the `copyto`, which can be iterated, and we pipe it directly into
# the `copyfrom`.
logger.info("Streaming the data from source to destination...")
response = copy_src_client.copyto('COPY %s TO STDOUT' % TABLE_NAME)
result = copy_dst_client.copyfrom('COPY %s FROM STDIN' % TABLE_NAME, response)
logger.info('Result: {}'.format(result))
github CartoDB / cartoframes / cartoframes / lib / context / api_context.py View on Github external
def __init__(self, credentials):
        self.auth_client = APIKeyAuthClient(
            base_url=credentials.base_url,
            api_key=credentials.api_key,
            session=credentials.session,
            client_id='cartoframes_{}'.format(__version__),
            user_agent='cartoframes_{}'.format(__version__)
        )
        self.sql_client = SQLClient(self.auth_client)
        self.copy_client = CopySQLClient(self.auth_client)
        self.batch_sql_client = BatchSQLClient(self.auth_client)
github CartoDB / cartoframes / cartoframes / core / managers / context_manager.py View on Github external
def __init__(self, credentials):
        self.credentials = credentials or get_default_credentials()
        check_credentials(self.credentials)

        self.auth_client = _create_auth_client(self.credentials)
        self.sql_client = SQLClient(self.auth_client)
        self.copy_client = CopySQLClient(self.auth_client)
        self.batch_sql_client = BatchSQLClient(self.auth_client)
github CartoDB / cartoframes / cartoframes / context.py View on Github external
self.creds = Credentials(creds=creds, key=api_key, base_url=base_url)
        self.auth_client = APIKeyAuthClient(
            base_url=self.creds.base_url(),
            api_key=self.creds.key(),
            session=session,
            client_id='cartoframes_{}'.format(__version__),
            user_agent='cartoframes_{}'.format(__version__)
        )
        self.auth_api_client = AuthAPIClient(
            base_url=self.creds.base_url(),
            api_key=self.creds.key(),
            session=session
        )
        self.sql_client = SQLClient(self.auth_client)
        self.copy_client = CopySQLClient(self.auth_client)
        self.batch_sql_client = BatchSQLClient(self.auth_client)
        self.creds.username(self.auth_client.username)
        self._is_authenticated()
        self.is_org = self._is_org_user()

        self._map_templates = {}
        self._srcdoc = None
        self._verbose = verbose
github CartoDB / carto-python / examples / duplicate_table.py View on Github external
logger.info(create_table_no_seqs)

# Create the table in the destination account
logger.info('Creating the table in the destination account...')
res = sql_dst_client.send(create_table_no_seqs)
logger.info('Response: {}'.format(res))

# Cartodbfy the table (this is optional)
logger.info("Cartodbfy'ing the destination table...")
res = sql_dst_client.send(
    "SELECT CDB_CartodbfyTable(current_schema, '%s')" % TABLE_NAME
)
logger.info('Response: {}'.format(res))

# Create COPY clients
copy_src_client = CopySQLClient(auth_src_client)
copy_dst_client = CopySQLClient(auth_dst_client)

# COPY (streaming) the data from the source to the dest table. We use
# here all the COPY defaults. Note that we take the `response` from
# the `copyto`, which can be iterated, and we pipe it directly into
# the `copyfrom`.
logger.info("Streaming the data from source to destination...")
response = copy_src_client.copyto('COPY %s TO STDOUT' % TABLE_NAME)
result = copy_dst_client.copyfrom('COPY %s FROM STDIN' % TABLE_NAME, response)
logger.info('Result: {}'.format(result))