Merge pull request #301 from USEPA/v1.3-release

V1.3 release

Author: catherinebirney

.github/workflows/compare_single_FBS.yml

@@ -12,21 +12,22 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
     strategy:
       fail-fast: false
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: "3.10"
 
     - name: Update pip
       run: |
         python -VV
         python -m pip install --upgrade pip setuptools wheel
+        pip install pytest
 
     # install package & dependencies
     - name: Install package and dependencies
@@ -42,14 +43,14 @@ jobs:
 
     - name: Upload files
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3.1.1
       with:
         # Artifact name
         name: FBS diff files
         # A file, directory or wildcard patter that describes what to upload
-        path: | # uses local user data dir for ubuntu
-          ~/.local/share/flowsa/FBSComparisons/*
-          ~/.local/share/flowsa/FlowBySector/*
-          ~/.local/share/stewi/Log/*
+        path: | # uses local user data dir for macos
+          ~/Library/Application Support/flowsa/FBSComparisons/*
+          ~/Library/Application Support/flowsa/FlowBySector/*
+          ~/Library/Application Support/flowsa/Log/*
         if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
         # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.

.github/workflows/generate_FBS.yml

@@ -10,14 +10,14 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
     strategy:
       fail-fast: false
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: "3.10"
 
@@ -38,13 +38,13 @@ jobs:
 
     - name: Upload files
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v3.1.1
       with:
         # Artifact name
         name: FlowBySector
         # A file, directory or wildcard patter that describes what to upload
         path: | # uses local user data dir for ubuntu
-          ~/.local/share/flowsa/FlowBySector/*
-          ~/.local/share/stewi/Log/*
+          ~/Library/Application Support/flowsa/FlowBySector/*
+          ~/Library/Application Support/flowsa/Log/*
         if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
         # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.

.github/workflows/python-app.yml

@@ -34,11 +34,11 @@ jobs:
         py-version: ['3.8', '3.9', '3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     # general Python setup
     - name: Set up Python ${{ matrix.py-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.py-version }}
 

.github/workflows/test_methods.yml

@@ -12,12 +12,12 @@ on:
 
 jobs:
   FBA_testing:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: "3.10"
 
@@ -37,12 +37,12 @@ jobs:
         python flowsa/test_methods.py
 
   FBS_testing:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: "3.10"
 
@@ -61,19 +61,22 @@ jobs:
       id: FBS
       if: always() # Proceed even if Test FBA fails
       run: |
-        python flowsa/test_FBS_against_remote.py
+        for m in $(python flowsa/test_FBS_against_remote.py list)
+        do
+          python flowsa/test_FBS_against_remote.py "$m"
+        done
 
     - name: Upload csv files
       # if: steps.FBS.outcome == 'success' # Upload csv only if step succeeds
       if: always()
-      uses: actions/upload-artifact@v2.3.0
+      uses: actions/upload-artifact@v3.1.1
       with:
         # Artifact name
         name: FBS diff files
         # A file, directory or wildcard patter that describes what to upload
         path: |
-            ~/.local/share/flowsa/FBSComparisons/*
-            ~/.local/share/flowsa/FlowBySector/*
-            ~/.local/share/flowsa/Log/*
+            ~/Library/Application Support/flowsa/FBSComparisons/*
+            ~/Library/Application Support/flowsa/FlowBySector/*
+            ~/Library/Application Support/flowsa/Log/*
         if-no-files-found: warn # 'warn' or 'ignore' are also available, defaults to `warn`
         # retention-days: 5 # cannot exceed the retention limit set by the repository, organization, or enterprise.

examples/external_paths.env.example

@@ -0,0 +1,9 @@
+"""
+To access a file stored externally from FLOWSA, users can add a filepath to
+"external_paths.env" and store the .env in the MODULEPATH defined in
+flowsa/settings.py.
+
+"""
+WARM-USEEIO_MaterialCrosswalk = "user-path/WARM-USEEIOdocs/data/MaterialCrosswalk.csv"
+
+WARM-USEEIO_Materials = "user-path/git_projects/WARM-USEEIOdocs/data/Materials.csv"

examples/generate_data_visualization.py

@@ -7,11 +7,12 @@
 """
 
 import flowsa
+import flowsa.datavisualization as dv
 from flowsa.settings import plotoutputpath
 import matplotlib.pyplot as plt
 
 
-# Produce facet graph of resources associated with cropland sectors
+########## Produce facet graph of resources associated with cropland sectors ##########
 sectors = ['112']
 sector_length_display = 6
 plottype = 'facet_graph'
@@ -20,16 +21,16 @@
                'Employment 2017': 'Employment_national_2017'}
 
 
-flowsa.generateFBSplot(method_dict, plottype,
-                       sector_length_display=sector_length_display,
-                       sectors_to_include=sectors,
-                       plot_title='Direct Resource Use for Livestock'
-                       )
+dv.FBSscatterplot(method_dict, plottype,
+                  sector_length_display=sector_length_display,
+                  sectors_to_include=sectors,
+                  plot_title='Direct Resource Use for Livestock'
+                  )
 # Can manually adjust the figure pop up before saving
 plt.savefig(f"{plotoutputpath}livestock_resource_use.png", dpi=300)
 
 
-# Compare the results between water method 1 and method 2
+########## Compare the results between water method 1 and method 2 ##########
 sectors = ['21']
 sector_length_display = 6
 plottype = 'method_comparison'
@@ -45,3 +46,52 @@
                        )
 # Can manually adjust the figure pop up before saving
 plt.savefig(f"{plotoutputpath}mining_water_comp.png", dpi=300)
+
+
+########## Compare food waste flows via Sankey ##########
+methodnames = ['Food_Waste_national_2018_m1', 'Food_Waste_national_2018_m2']
+target_sector_level = 'NAICS_2'
+target_subset_sector_level = {
+    'Food_Waste_national_2018_m1': {'NAICS_4': ['2212'],
+                                    'NAICS_6': ['62421', '31111', '32411',
+                                                '56221', '62421', '115112',
+                                                '22132'],
+                                    'NAICS_7': ['562212', '562219']
+                                    },
+    'Food_Waste_national_2018_m2': {
+        'SectorConsumedBy': {'NAICS_6': ['115112', '22132','311119',
+                                         '32411', '562213','62421'],
+                             'NAICS_7': ['562212', '562219']
+                               }}
+}
+# set domain to scale sankey diagrams
+domain_dict = {0: {'x': [0.01, 0.49], 'y': [0, 1]},
+               1: {'x': [.51, 0.99], 'y': [.12, .88]}
+               }
+
+dv.generateSankeyDiagram(
+    methodnames,
+    target_sector_level=target_sector_level,
+    target_subset_sector_level=target_subset_sector_level,
+    use_sectordefinition=True,
+    sectors_to_include=None,
+    fbsconfigpath=None,
+    orientation='horizontal',
+    domain_dict=domain_dict,
+    value_label_format='brackets',
+    subplot_titles=['m1', 'm2'],
+    filename='FoodWasteSankey'
+)
+
+
+####### GHG Bar Chart ############
+# Option 1 - GHG emissions by GHG
+dv.stackedBarChart('GHG_national_2018_m1',
+                   sectors_to_include=['111110', '112120', '325312'],
+                   filename='GHGEmissions')
+
+# Option 2 - specify indicator, much have LCIAformatter installed
+# https://github.com/USEPA/LCIAformatter
+dv.stackedBarChart('GHG_national_2018_m1', impact_cat='Global warming',
+                   sectors_to_include=['111110', '112120', '325312'],
+                   filename='GHGEmissionsGlobalWarming')

flowsa/__init__.py

@@ -30,7 +30,7 @@
 import flowsa.flowbyactivity
 import flowsa.flowbysector
 from flowsa.bibliography import generate_fbs_bibliography
-from flowsa.datavisualization import plotFBSresults
+from flowsa.datavisualization import FBSscatterplot
 
 
 def getFlowByActivity(datasource, year, flowclass=None, geographic_level=None,
@@ -39,7 +39,8 @@ def getFlowByActivity(datasource, year, flowclass=None, geographic_level=None,
     Retrieves stored data in the FlowByActivity format
     :param datasource: str, the code of the datasource.
     :param year: int, a year, e.g. 2012
-    :param flowclass: str, a 'Class' of the flow. Optional. E.g. 'Water'
+    :param flowclass: str or list, a 'Class' of the flow. Optional. E.g.
+    'Water' or ['Employment', 'Chemicals']
     :param geographic_level: str, a geographic level of the data.
                              Optional. E.g. 'national', 'state', 'county'.
     :param download_FBA_if_missing: bool, if True will attempt to load from
@@ -81,7 +82,11 @@ def getFlowByActivity(datasource, year, flowclass=None, geographic_level=None,
 
     # Address optional parameters
     if flowclass is not None:
-        fba = fba[fba['Class'] == flowclass]
+        # subset df by single class or list of classes
+        if isinstance(flowclass, str):
+            fba = fba[fba['Class'] == flowclass]
+        else:
+            fba = fba[fba['Class'].isin(flowclass)]
     # if geographic level specified, only load rows in geo level
     if geographic_level is not None:
         fba = filter_by_geoscale(fba, geographic_level)
@@ -234,5 +239,5 @@ def generateFBSplot(method_dict, plottype, sector_length_display=None,
     :return: graphic displaying results of FBS models
     """
 
-    plotFBSresults(method_dict, plottype, sector_length_display,
+    FBSscatterplot(method_dict, plottype, sector_length_display,
                    sectors_to_include, plot_title)

flowsa/allocation.py

@@ -11,7 +11,7 @@
 from flowsa.settings import log, vLogDetailed
 from flowsa.dataclean import replace_NoneType_with_empty_cells, \
     replace_strings_with_NoneType
-from flowsa.flowbyfunctions import sector_aggregation, \
+from flowsa.flowbyfunctions import sector_aggregation, aggregator, \
     sector_disaggregation, subset_and_merge_df_by_sector_lengths
 
 
@@ -343,20 +343,26 @@ def equal_allocation(fba_load):
     """
     from flowsa.flowbyfunctions import assign_columns_of_sector_levels
 
+    # aggregate df
+    aggcols = list(fba_load.select_dtypes(include=['object', 'int']).columns)
+    fba = aggregator(fba_load, aggcols)
+
     # first check that all sector lengths are the same
-    dfc = assign_columns_of_sector_levels(fba_load)
+    dfc = assign_columns_of_sector_levels(fba)
     # if duplicated rows, keep assignment to most specific sectors because
     # data should already be at final assignment lengths if equally
     # allocating and because not manipulating the loaded dataset, but rather
     # just checking that all sector lengths match for an activity
     duplicate_cols = [e for e in dfc.columns if e not in [
         'SectorProducedByLength', 'SectorConsumedByLength']]
-    duplicate_df = dfc[dfc.duplicated(duplicate_cols)]
+    duplicate_df = dfc[dfc.duplicated(duplicate_cols, keep=False)]
     if len(duplicate_df) > 0:
         log.info('Dropping rows duplicated due to assigning sector lengths '
                  'for ambiguous sectors. Keeping sector length assignments '
                  'to most specific sectors.')
-        dfc = dfc[dfc.duplicated(duplicate_cols, keep='first')]
+        dfc = dfc.sort_values(['SectorProducedByLength',
+                               'SectorConsumedByLength'])
+        dfc = dfc[~dfc.duplicated(duplicate_cols, keep='last')]
 
     # Before equally allocating, check that each activity is being allocated
     # to sectors of the same length
@@ -367,19 +373,20 @@ def equal_allocation(fba_load):
     if len(df_dup) > 1:
         log.error('Cannot equally allocate because sector lengths vary. All '
                   'sectors must be the same sector level.')
+        return fba
 
     # create groupby cols by which to determine allocation
-    fba_cols = fba_load.select_dtypes([object]).columns.to_list()
+    fba_cols = fba.select_dtypes([object]).columns.to_list()
     groupcols = [e for e in fba_cols if e not in
                  ['SectorProducedBy', 'SectorConsumedBy', 'Description']]
     # create counts of rows
-    df_count = fba_load.groupby(
+    df_count = fba.groupby(
         groupcols, as_index=False, dropna=False).size()
     df_count = replace_NoneType_with_empty_cells(df_count)
 
     # merge dfs, replace cells with empty strings to ensure merge occurs
     # correctly
-    fba = replace_NoneType_with_empty_cells(fba_load)
+    fba = replace_NoneType_with_empty_cells(fba)
     dfm = fba.merge(df_count, how='outer', on=groupcols)
     # calc new flowamounts
     dfm['FlowAmount'] = dfm['FlowAmount'] / dfm['size']