--
-- Step 01 - create schemas
--

-- stage schema
IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = 'STAGE')
    EXEC sp_executesql N'CREATE SCHEMA STAGE'
GO

-- stocks schema
IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = 'STOCKS')
    EXEC sp_executesql N'CREATE SCHEMA STOCKS'
GO

-- show user defined schemas
SELECT * FROM sys.schemas WHERE name in ('STOCKS', 'STAGE')
GO


--
-- Step 02 - drop tables
--

-- remove staging table
DROP TABLE IF EXISTS STAGE.SNP500
GO

-- remove stocks table
DROP TABLE IF EXISTS STOCKS.SNP500
GO


--
-- Step 03 - create tables
--

-- create staging table
CREATE TABLE STAGE.SNP500
(
  ST_SYMBOL VARCHAR(32) NOT NULL,
  ST_DATE DATE NOT NULL,
  ST_OPEN REAL NULL,
  ST_HIGH REAL NULL,
  ST_LOW REAL NULL,
  ST_CLOSE REAL NULL,
  ST_ADJ_CLOSE REAL NULL,
  ST_VOLUME BIGINT NULL
);
GO

-- create stocks table
CREATE TABLE STOCKS.SNP500
(
  ST_SYMBOL VARCHAR(32) NOT NULL,
  ST_DATE DATE NOT NULL,
  ST_AVG_OPEN REAL NULL,
  ST_AVG_HIGH REAL NULL,
  ST_AVG_LOW REAL NULL,
  ST_AVG_CLOSE REAL NULL,
  ST_AVG_ADJ_CLOSE REAL NULL,
  ST_AVG_VOLUME BIGINT NULL
);
GO


--
-- Step 03 - create tables
--

-- show user defined tables
SELECT 
    s.name AS SCHEMA_NM,
    t.name as TABLE_NM
FROM 
    sys.tables AS t
JOIN 
    sys.schemas AS s
ON 
    t.schema_id = s.schema_id
GO


--
-- Step 04 - fast load of detailed data
--

-- copy into statment
COPY INTO STAGE.SNP500
FROM
  'https://sa4adls2030.dfs.core.windows.net/raw/stocks2/YR2013/*.CSV',
  'https://sa4adls2030.dfs.core.windows.net/raw/stocks2/YR2014/*.CSV',
  'https://sa4adls2030.dfs.core.windows.net/raw/stocks2/YR2015/*.CSV',
  'https://sa4adls2030.dfs.core.windows.net/raw/stocks2/YR2016/*.CSV',
  'https://sa4adls2030.dfs.core.windows.net/raw/stocks2/YR2017/*.CSV'
WITH 
(
    FILE_TYPE = 'CSV',
    CREDENTIAL=(IDENTITY= 'Storage Account Key', SECRET='tA5k3pJRZv16vRQOOV80D7moaRLnE5rNF5gnjPU3YR6FrKcLz6uCHaaLfd9PN4mCk7BNvY6RsSyiFmQSd9MCwA=='),
    FIELDQUOTE = '"',
    FIELDTERMINATOR=',',
    ROWTERMINATOR='0x0A',
    ENCODING = 'UTF8',
    FIRSTROW = 2
)
GO


--
-- Step 05 - load summary data
--

-- insert aggregated data into table
INSERT INTO STOCKS.SNP500
SELECT
  ST_SYMBOL,
  CAST(DATEADD(m, DATEDIFF(m, 0, ST_DATE), 0) AS DATE) AS ST_DATE,
  AVG(ST_OPEN) AS ST_AVG_OPEN,
  AVG(ST_HIGH) AS ST_AVG_HIGH,
  AVG(ST_LOW) AS ST_AVG_LOW,
  AVG(ST_CLOSE) AS ST_AVG_CLOSE,
  AVG(ST_ADJ_CLOSE) AS ST_AVG_ADJ_CLOSE,
  AVG(ST_VOLUME) AS ST_AVG_VOLUME
FROM
  STAGE.SNP500
GROUP BY
  ST_SYMBOL,
  CAST(DATEADD(m, DATEDIFF(m, 0, ST_DATE), 0) AS DATE)
GO


--
-- Step 06 - obtain record counts
--

-- detail vs summary
SELECT 'SUMMARY' AS TABLE_TYPE, COUNT(*) AS TABLE_ROWS FROM STOCKS.SNP500
UNION
SELECT 'DETAIL' AS TABLE_TYPE, COUNT(*) AS TABLE_ROWS FROM STAGE.SNP500
GO