Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_internal_copy_from(self, mocker):
# Given
from shapely.geometry import Point
mocker.patch('cartoframes.core.managers.context_manager._create_auth_client')
mock = mocker.patch.object(CopySQLClient, 'copyfrom')
cdf = CartoDataFrame({'A': [1, 2], 'B': [Point(0, 0), Point(1, 1)]})
columns = [
ColumnInfo('A', 'a', 'bigint', False),
ColumnInfo('B', 'b', 'geometry', True)
]
# When
cm = ContextManager(self.credentials)
cm._copy_from(cdf, 'table_name', columns)
# Then
assert mock.call_args[0][0] == '''
COPY table_name(a,b) FROM stdin WITH (FORMAT csv, DELIMITER '|', NULL '__null');
'''.strip()
assert list(mock.call_args[0][1]) == [
b'1|0101000020E610000000000000000000000000000000000000\n',
def copy_client(api_key_auth_client_usr):
return CopySQLClient(api_key_auth_client_usr)
logger.error('You need to provide valid credentials, run with '
'-h parameter for details')
sys.exit(1)
# Create and cartodbfy a table
sqlClient = SQLClient(auth_client)
sqlClient.send("""
CREATE TABLE IF NOT EXISTS copy_example (
the_geom geometry(Geometry,4326),
name text,
age integer
)
""")
sqlClient.send("SELECT CDB_CartodbfyTable(current_schema, 'copy_example')")
copyClient = CopySQLClient(auth_client)
# COPY FROM example
logger.info("COPY'ing FROM file...")
query = ('COPY copy_example (the_geom, name, age) '
'FROM stdin WITH (FORMAT csv, HEADER true)')
result = copyClient.copyfrom_file_path(query, 'files/copy_from.csv')
logger.info('result = %s' % result)
# COPY TO example with pandas DataFrame
logger.info("COPY'ing TO pandas DataFrame...")
query = 'COPY copy_example TO stdout WITH (FORMAT csv, HEADER true)'
result = copyClient.copyto_stream(query)
df = pd.read_csv(result)
logger.info(df.head())
# Truncate the table to make this example repeatable
BATCH_TERMINAL_STATES = ['done', 'failed', 'cancelled', 'unknown']
logger.info("Dropping, re-creating and cartodbfy'ing the table through the Batch API...")
batchSQLClient = BatchSQLClient(auth_client)
job = batchSQLClient.create(LIST_OF_SQL_QUERIES)
while not job['status'] in BATCH_TERMINAL_STATES:
time.sleep(1)
job = batchSQLClient.read(job['job_id'])
if job['status'] != 'done':
logger.error('Could not create and cartodbfy table')
logger.error(job['failed_reason'])
sys.exit(1)
logger.info("Table created and cartodbfy'ed")
copyClient = CopySQLClient(auth_client)
# COPY FROM example
logger.info("COPY'ing the table FROM the file...")
query = 'COPY tl_2014_census_tracts (the_geom,statefp,countyfp,tractce,geoid,name,cartodb_id,aland,awater,created_at,updated_at,mtfcc) FROM stdin WITH (FORMAT csv, HEADER true)'
input_file = '/home/rtorre/src/cartodb-tests/sql_copy_perf/tl_2014_census_tracts.csv'
result = copyClient.copyfrom_file_path(query, input_file)
logger.info(result)
logger.info("Table populated")
def __init__(self, credentials):
self.auth_client = APIKeyAuthClient(
base_url=credentials.base_url,
api_key=credentials.api_key,
session=credentials.session,
client_id='cartoframes_{}'.format(__version__),
user_agent='cartoframes_{}'.format(__version__)
)
self.sql_client = SQLClient(self.auth_client)
self.copy_client = CopySQLClient(self.auth_client)
self.batch_sql_client = BatchSQLClient(self.auth_client)
# Create the table in the destination account
logger.info('Creating the table in the destination account...')
res = sql_dst_client.send(create_table_no_seqs)
logger.info('Response: {}'.format(res))
# Cartodbfy the table (this is optional)
logger.info("Cartodbfy'ing the destination table...")
res = sql_dst_client.send(
"SELECT CDB_CartodbfyTable(current_schema, '%s')" % TABLE_NAME
)
logger.info('Response: {}'.format(res))
# Create COPY clients
copy_src_client = CopySQLClient(auth_src_client)
copy_dst_client = CopySQLClient(auth_dst_client)
# COPY (streaming) the data from the source to the dest table. We use
# here all the COPY defaults. Note that we take the `response` from
# the `copyto`, which can be iterated, and we pipe it directly into
# the `copyfrom`.
logger.info("Streaming the data from source to destination...")
response = copy_src_client.copyto('COPY %s TO STDOUT' % TABLE_NAME)
result = copy_dst_client.copyfrom('COPY %s FROM STDIN' % TABLE_NAME, response)
logger.info('Result: {}'.format(result))
def __init__(self, credentials):
self.auth_client = APIKeyAuthClient(
base_url=credentials.base_url,
api_key=credentials.api_key,
session=credentials.session,
client_id='cartoframes_{}'.format(__version__),
user_agent='cartoframes_{}'.format(__version__)
)
self.sql_client = SQLClient(self.auth_client)
self.copy_client = CopySQLClient(self.auth_client)
self.batch_sql_client = BatchSQLClient(self.auth_client)
def __init__(self, credentials):
self.credentials = credentials or get_default_credentials()
check_credentials(self.credentials)
self.auth_client = _create_auth_client(self.credentials)
self.sql_client = SQLClient(self.auth_client)
self.copy_client = CopySQLClient(self.auth_client)
self.batch_sql_client = BatchSQLClient(self.auth_client)
self.creds = Credentials(creds=creds, key=api_key, base_url=base_url)
self.auth_client = APIKeyAuthClient(
base_url=self.creds.base_url(),
api_key=self.creds.key(),
session=session,
client_id='cartoframes_{}'.format(__version__),
user_agent='cartoframes_{}'.format(__version__)
)
self.auth_api_client = AuthAPIClient(
base_url=self.creds.base_url(),
api_key=self.creds.key(),
session=session
)
self.sql_client = SQLClient(self.auth_client)
self.copy_client = CopySQLClient(self.auth_client)
self.batch_sql_client = BatchSQLClient(self.auth_client)
self.creds.username(self.auth_client.username)
self._is_authenticated()
self.is_org = self._is_org_user()
self._map_templates = {}
self._srcdoc = None
self._verbose = verbose
logger.info(create_table_no_seqs)
# Create the table in the destination account
logger.info('Creating the table in the destination account...')
res = sql_dst_client.send(create_table_no_seqs)
logger.info('Response: {}'.format(res))
# Cartodbfy the table (this is optional)
logger.info("Cartodbfy'ing the destination table...")
res = sql_dst_client.send(
"SELECT CDB_CartodbfyTable(current_schema, '%s')" % TABLE_NAME
)
logger.info('Response: {}'.format(res))
# Create COPY clients
copy_src_client = CopySQLClient(auth_src_client)
copy_dst_client = CopySQLClient(auth_dst_client)
# COPY (streaming) the data from the source to the dest table. We use
# here all the COPY defaults. Note that we take the `response` from
# the `copyto`, which can be iterated, and we pipe it directly into
# the `copyfrom`.
logger.info("Streaming the data from source to destination...")
response = copy_src_client.copyto('COPY %s TO STDOUT' % TABLE_NAME)
result = copy_dst_client.copyfrom('COPY %s FROM STDIN' % TABLE_NAME, response)
logger.info('Result: {}'.format(result))