Source code for util.spatial

from util import BaseUtil



class SpatialUtil:




    @staticmethod
    def download_from_esri(url:str, fields=None, where=None, limit=None, **kwargs):
        """A util to download data from an ESRI endpoints. This is a wrapper around geopandas.read_file that handles the ESRI API's max record count.
        It loads the data in chunks and concatenates them into a single GeoDataFrame. The final GeoDataFrame is returned and can get large if the limit is not set. One approach to do this is to set the limit to a small number and then filter the data in the where clause. or order them by a field and set the limit to the number of records you want to download.

        Note: This function uses the pyogrio engine to read the data. This is a faster engine than the default fiona engine. To use this engine, you need to install the pyogrio package. You can install it using `pip install pyogrio`.
        Note: It uses `get` request. it might reject your request if you send large request body geometries. can use post for such cases but it is not implemented here. ask Majid for help if you need it.

        :param url: Esri endpoint url for example https://services.arcgis.com/8DAUcrpQcpyLMznu/arcgis/rest/services/COVID19_Cases_by_Zip_Code/FeatureServer/0
        :type url: str
        :param fields: the list of the fields to return, defaults to None which loads all fields by passing `*` to the API
        :type fields: _type_, optional
        :param where: the filter criteria, defaults to None which means no filter and it sends `1=1` to the API
        :type where: _type_, optional
        :param limit: the max number of records to get, defaults to all the values in the feaure layer
        :type limit: _type_, optional
        :return: A pandas DataFrame with the data
        :rtype: _type_

        :example:
        ```python
            url = "https://pcrdgis.agriculture.purdue.edu/arcgis/rest/services/DDI/Digital_Divide_Index_COUNTY_USA_2020/MapServer/0"
            gdf=SpatialUtil.download_from_esri(url=url)
        ```
        """
        import geopandas as gpd
        import pandas as pd
        import json
        from tempfile import NamedTemporaryFile

        gpd.options.io_engine = "pyogrio" 
        # Get the max record count
        metadata = BaseUtil.get_request(url=url,params=dict(f="pjson")).json()
        max_record_count = metadata["maxRecordCount"]

        # default behavior matches all features
        if where is None:
            where = "1=1"
        if fields is None:
            fields = "*"
        else:
            fields = ", ".join(fields)

        # extract object IDs of features
        queryURL = f"{url}/query"

        # Get the total record count
        params = dict(where=where, returnCountOnly="true", f="json")
        response = BaseUtil.get_request(url=queryURL, params=params)

        
        total_size = BaseUtil.get_json_safely(response)["count"]

        # Check the limit
        if limit is not None:
            total_size = limit

        # params for this request
        resultOffset = 0
        params = dict(
            f="json",
            outSR="4326",
            outFields=fields,
            resultOffset=resultOffset,
            where=where,
            **kwargs,
        )


        calls = total_size // max_record_count
        if calls > 10:
            print(
                f"Long download time — total download will require {calls} separate requests"
            )
        out = []
        while params["resultOffset"] < total_size:

            remaining = total_size - params["resultOffset"]
            if remaining < max_record_count:
                params["resultRecordCount"] = remaining

            # get raw features
            response = BaseUtil.get_request(url=queryURL, params=params)
            json_content = BaseUtil.get_json_safely(response)

            with NamedTemporaryFile(suffix=".json",mode="w+") as json_file:

                # write to file
                json.dump(json_content, json_file)
                json_file.flush()
                
                gdf=gpd.read_file(json_file.name, driver="ESRIJSON",crs="EPSG:4326")
                out.append(gdf)
            params["resultOffset"] += len(out[-1])

        return pd.concat(out, axis=0).reset_index(drop=True)