diff --git a/Makefile b/Makefile index 6bbf46f..39f9414 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ psql_command=psql -q today:=$(shell date "+%Y%m%d") -gias_filename:=edubasealldata${today}.csv -fixed_filename=edubasealldata${today}-fixed.csv +gias_schools_file:=edubasealldata${today}.csv +gias_links_file:=links_edubasealldata${today}.csv +fixed_school_filename=edubasealldata${today}-fixed.csv +fixed_links_filename=links_edubasealldata${today}-fixed.csv database_name=gias data_dir=tmp @@ -19,10 +21,18 @@ refresh: drop_database \ create_views \ refresh_views -download_gias_data: +download_gias_data: clear_tmp_csvs download_schools_data download_school_links_data + +clear_tmp_csvs: rm -f tmp/*.csv - wget https://ea-edubase-api-prod.azurewebsites.net/edubase/downloads/public/${gias_filename} --directory-prefix=${data_dir} - iconv -f ISO8859-1 -t UTF-8 tmp/${gias_filename} > tmp/${fixed_filename} + +download_schools_data: + wget https://ea-edubase-api-prod.azurewebsites.net/edubase/downloads/public/${gias_schools_file} --directory-prefix=${data_dir} + iconv -f ISO8859-1 -t UTF-8 tmp/${gias_schools_file} > tmp/${fixed_school_filename} + +download_school_links_data: + wget https://ea-edubase-api-prod.azurewebsites.net/edubase/downloads/public/${gias_links_file} --directory-prefix=${data_dir} + iconv -f ISO8859-1 -t UTF-8 tmp/${gias_links_file} > tmp/${fixed_links_filename} drop_database: dropdb --if-exists ${database_name} @@ -39,6 +49,7 @@ create_holding_tables: ${psql_command} ${database_name} < ddl/tables/create_deprivation_pupil_premium_raw.sql ${psql_command} ${database_name} < ddl/tables/geo/create_electoral_regions_raw.sql ${psql_command} ${database_name} < ddl/tables/geo/create_local_authority_districts_raw.sql + ${psql_command} ${database_name} < ddl/tables/create_school_links_raw.sql create_types: ${psql_command} ${database_name} < ddl/types/establishment.sql @@ -47,18 +58,21 @@ create_types: ${psql_command} ${database_name} < ddl/types/phase.sql ${psql_command} ${database_name} < ddl/types/rural_urban_classification.sql ${psql_command} ${database_name} < ddl/types/government_office_regions.sql + ${psql_command} ${database_name} < ddl/types/link_types.sql create_data_tables: ${psql_command} ${database_name} < ddl/tables/create_schools.sql ${psql_command} ${database_name} < ddl/tables/create_deprivation_pupil_premium.sql ${psql_command} ${database_name} < ddl/tables/geo/create_regions.sql ${psql_command} ${database_name} < ddl/tables/geo/create_local_authorities.sql + ${psql_command} ${database_name} < ddl/tables/create_school_links.sql create_views: ${psql_command} ${database_name} < ddl/views/open_schools.sql populate_holding_tables: - ${psql_command} ${database_name} --command "\copy schools_raw from 'tmp/${fixed_filename}' with csv header" + ${psql_command} ${database_name} --command "\copy schools_raw from 'tmp/${fixed_school_filename}' with csv header" + ${psql_command} ${database_name} --command "\copy school_links_raw from 'tmp/${fixed_links_filename}' with csv header" ${psql_command} ${database_name} < dml/import_email_addresses_raw.sql ${psql_command} ${database_name} < dml/import_deprivation_pupil_premium_raw.sql ${psql_command} ${database_name} < dml/geo/import_electoral_regions.sql @@ -67,6 +81,7 @@ populate_holding_tables: drop_holding_tables: ${psql_command} ${database_name} < ddl/tables/drop_schools_raw.sql ${psql_command} ${database_name} < ddl/tables/drop_email_addresses_raw.sql + ${psql_command} ${database_name} < ddl/tables/drop_school_links_raw.sql ${psql_command} ${database_name} < ddl/tables/drop_deprivation_pupil_premium_raw.sql ${psql_command} ${database_name} < ddl/tables/geo/drop_electoral_regions_raw.sql ${psql_command} ${database_name} < ddl/tables/geo/drop_local_authority_districts_raw.sql @@ -76,6 +91,7 @@ populate_data_tables: ${psql_command} ${database_name} < dml/import_deprivation_pupil_premium.sql ${psql_command} ${database_name} < dml/geo/import_regions.sql ${psql_command} ${database_name} < dml/geo/import_districts.sql + ${psql_command} ${database_name} < dml/import_school_links.sql refresh_views: ${psql_command} ${database_name} < ddl/refresh/refresh_open_schools.sql diff --git a/ddl/tables/create_school_links.sql b/ddl/tables/create_school_links.sql new file mode 100644 index 0000000..c9aa051 --- /dev/null +++ b/ddl/tables/create_school_links.sql @@ -0,0 +1,8 @@ +drop table if exists school_links; + +create table school_links ( + urn1 integer references schools(urn), + urn2 integer references schools(urn), + type link_type, + date date +); diff --git a/ddl/tables/create_school_links_raw.sql b/ddl/tables/create_school_links_raw.sql new file mode 100644 index 0000000..5084a01 --- /dev/null +++ b/ddl/tables/create_school_links_raw.sql @@ -0,0 +1,7 @@ +CREATE TABLE "school_links_raw" ( + "URN" VARCHAR NOT NULL, + "LinkURN" VARCHAR NOT NULL, + "LinkName" VARCHAR, + "LinkType" VARCHAR, + "LinkEstablishedDate" VARCHAR +); diff --git a/ddl/tables/drop_school_links_raw.sql b/ddl/tables/drop_school_links_raw.sql new file mode 100644 index 0000000..b668887 --- /dev/null +++ b/ddl/tables/drop_school_links_raw.sql @@ -0,0 +1 @@ +DROP TABLE schools_links_raw; diff --git a/ddl/types/link_types.sql b/ddl/types/link_types.sql new file mode 100644 index 0000000..0519325 --- /dev/null +++ b/ddl/types/link_types.sql @@ -0,0 +1,21 @@ +drop type if exists link_type; + +create type link_type as enum ( + 'Closure', + 'Expansion', + 'Merged - change in age range', + 'Merged - expansion in school capacity and changer in age range', + 'Merged - expansion of school capacity', + 'Other', + 'Predecessor', + 'Predecessor - Split School', + 'Predecessor - amalgamated', + 'Predecessor - merged', + 'Result of Amalgamation', + 'Sixth Form Centre Link', + 'Sixth Form Centre School', + 'Successor', + 'Successor - Split School', + 'Successor - amalgamated', + 'Successor - merged' +) diff --git a/dml/import_school_links.sql b/dml/import_school_links.sql new file mode 100644 index 0000000..663c6ed --- /dev/null +++ b/dml/import_school_links.sql @@ -0,0 +1,285 @@ +set datestyle to DMY; + +insert into school_links ( + urn1, + urn2, + type, + date +) +select + slr."URN"::integer, + slr."LinkURN"::integer, + slr."LinkType"::link_type, + case when (slr."LinkEstablishedDate" is null or slr."LinkEstablishedDate" = '') + then null + else + slr."LinkEstablishedDate"::date + end +from school_links_raw slr +where + slr."LinkURN"::integer not in ( + '151595', + '531429', + '152137', + '152138', + '151985', + '151015', + '151946', + '152263', + '152286', + '152287', + '151947', + '151809', + '149922', + '152267', + '151891', + '151353', + '515781', + '152167', + '152129', + '151638', + '152026', + '152257', + '152027', + '152240', + '152032', + '152033', + '152034', + '151020', + '152070', + '152080', + '152081', + '152258', + '152166', + '151599', + '152265', + '152293', + '152262', + '152014', + '152015', + '152193', + '152016', + '152017', + '152018', + '152264', + '151818', + '151819', + '151820', + '525028', + '525029', + '525037', + '152252', + '151033', + '152202', + '152203', + '152204', + '152019', + '152020', + '151674', + '149013', + '152273', + '152274', + '152088', + '152147', + '152192', + '152139', + '152140', + '151411', + '152253', + '149539', + '151512', + '152145', + '151790', + '151890', + '148909', + '151786', + '152271', + '152272', + '152134', + '152135', + '152261', + '151960', + '152176', + '149955', + '151958', + '151959', + '152260', + '152292', + '151957', + '152041', + '151579', + '149163', + '151414', + '152120', + '151415', + '145247', + '151813', + '151974', + '151975', + '151976', + '151977', + '152149', + '152173', + '151972', + '151973', + '151989', + '152168', + '151463', + '151785', + '151743', + '545256', + '152295', + '150499', + '150339', + '152201', + '151888', + '152071', + '152072', + '152073', + '151759', + '152074', + '152133', + '151889', + '151758', + '152087', + '148740', + '151604', + '152175', + '151778', + '152157', + '151220', + '152141', + '152067', + '152144', + '144876', + '152205', + '151930', + '151580', + '151339', + '151528', + '151983', + '151082', + '151925', + '151926', + '151927', + '152231', + '146577', + '146592', + '146608', + '146585', + '146604', + '151501', + '149980', + '152194', + '152266', + '151966', + '151967', + '152198', + '152199', + '152200', + '584235', + '152042', + '152195', + '151677', + '151678', + '152179', + '152183', + '152169', + '148923', + '152294', + '151754', + '151994', + '151713', + '152256', + '151986', + '151987', + '150093', + '151955', + '152268', + '152065', + '152066', + '151884', + '151885', + '152028', + '152029', + '151886', + '151887', + '152148', + '151761', + '151626', + '151577', + '147426', + '151578', + '151793', + '152283', + '152284', + '152285', + '152078', + '152142', + '152143', + '151852', + '152220', + '151815', + '152136', + '152196', + '152197', + '152219', + '151826', + '151827', + '151828', + '151587', + '151780', + '151781', + '151782', + '152269', + '525032', + '152060', + '152254', + '152255', + '516411', + '151981', + '151982', + '151932', + '152038', + '149690', + '140986', + '151755', + '585366', + '151980', + '151843', + '113934', + '103021', + '103020', + '116571', + '104970', + '122379', + '112073', + '111492', + '522902', + '152079', + '152239', + '104838', + '122919', + '125433', + '151753', + '150404', + '151854', + '146599', + '151685', + '151934', + '152259', + '152270', + '126131', + '131686', + '149325', + '151935', + '151196', + '152099', + '151425', + '151199', + '148007', + '146764', + '152105', + '151814' + ) +;