fetch.py 11.7 KB
Newer Older
Phyks's avatar
Phyks committed
1 2 3 4 5
# coding: utf-8
"""
This module contains all the code related to fetching and loading flats lists.
"""
from __future__ import absolute_import, print_function, unicode_literals
6
from builtins import str
Phyks's avatar
Phyks committed
7

8
import collections
Phyks's avatar
Phyks committed
9
import itertools
Phyks's avatar
Phyks committed
10 11 12
import json
import logging

13
from flatisfy import database
Phyks's avatar
Phyks committed
14
from flatisfy import tools
15
from flatisfy.constants import BACKENDS_BY_PRECEDENCE
16
from flatisfy.models import flat as flat_model
Phyks's avatar
Phyks committed
17 18 19 20

LOGGER = logging.getLogger(__name__)


Phyks's avatar
Phyks committed
21
try:
22
    from weboob.capabilities.housing import Query, POSTS_TYPES, HOUSE_TYPES
23
    from weboob.core.bcall import CallErrors
Phyks's avatar
Phyks committed
24 25 26 27 28 29 30 31
    from weboob.core.ouiboube import WebNip
    from weboob.tools.json import WeboobEncoder
except ImportError:
    LOGGER.error("Weboob is not available on your system. Make sure you "
                 "installed it.")
    raise


Phyks's avatar
Phyks committed
32
class WebOOBProxy(object):
Phyks's avatar
Phyks committed
33
    """
Phyks's avatar
Phyks committed
34
    Wrapper around WebOOB ``WebNip`` class, to fetch housing posts without
Phyks's avatar
Phyks committed
35 36 37 38 39
    having to spawn a subprocess.
    """
    @staticmethod
    def version():
        """
Phyks's avatar
Phyks committed
40
        Get WebOOB version.
Phyks's avatar
Phyks committed
41

Phyks's avatar
Phyks committed
42
        :return: The installed WebOOB version.
Phyks's avatar
Phyks committed
43 44 45
        """
        return WebNip.VERSION

Phyks's avatar
Phyks committed
46 47 48 49 50 51 52 53 54 55 56 57 58 59
    @staticmethod
    def restore_decimal_fields(flat):
        """
        Parse fields expected to be in Decimal type to float. They were dumped
        as str in the JSON dump process.

        :param flat: A flat dict.
        :return: A flat dict with Decimal fields converted to float.
        """
        for field in ["area", "cost", "rooms", "bedrooms", "price_per_meter"]:
            try:
                flat[field] = float(flat[field])
            except (TypeError, ValueError):
                flat[field] = None
Phyks's avatar
Phyks committed
60 61
            except KeyError:
                pass
Phyks's avatar
Phyks committed
62 63
        return flat

Phyks's avatar
Phyks committed
64 65
    def __init__(self, config):
        """
Phyks's avatar
Phyks committed
66
        Create a WebOOB handle and try to load the modules.
Phyks's avatar
Phyks committed
67 68 69

        :param config: A config dict.
        """
70 71
        # Default backends
        if not config["backends"]:
72
            backends = BACKENDS_BY_PRECEDENCE
73 74 75
        else:
            backends = config["backends"]

Phyks's avatar
Phyks committed
76 77 78 79 80 81 82 83 84 85
        # Create base WebNip object
        self.webnip = WebNip(modules_path=config["modules_path"])

        # Create backends
        self.backends = [
            self.webnip.load_backend(
                module,
                module,
                params={}
            )
86
            for module in backends
Phyks's avatar
Phyks committed
87 88 89 90 91 92 93 94 95 96
        ]

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.webnip.deinit()

    def build_queries(self, constraints_dict):
        """
Phyks's avatar
Phyks committed
97
        Build WebOOB ``weboob.capabilities.housing.Query`` objects from the
Phyks's avatar
Phyks committed
98
        constraints defined in the configuration. Each query has at most 3
99
        cities, to comply with housing websites limitations.
Phyks's avatar
Phyks committed
100 101

        :param constraints_dict: A dictionary of constraints, as defined in the
Phyks's avatar
Phyks committed
102
            config.
Phyks's avatar
Phyks committed
103
        :return: A list of WebOOB ``weboob.capabilities.housing.Query``
Phyks's avatar
Phyks committed
104
            objects. Returns ``None`` if an error occurred.
Phyks's avatar
Phyks committed
105 106
        """
        queries = []
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123

        # First, find all matching cities for the postal codes in constraints
        matching_cities = []
        for postal_code in constraints_dict["postal_codes"]:
            try:
                for city in self.webnip.do("search_city", postal_code):
                    matching_cities.append(city)
            except CallErrors as exc:
                # If an error occured, just log it
                LOGGER.error(
                    (
                        "An error occured while building query for "
                        "postal code %s: %s"
                    ),
                    postal_code,
                    str(exc)
                )
124 125 126 127

                if not matching_cities:
                    # If postal code gave no match, warn the user
                    LOGGER.warn(
Phyks's avatar
Phyks committed
128 129 130
                        "Postal code %s could not be matched with a city.",
                        postal_code
                    )
131

132 133 134 135 136 137 138 139 140
        # Remove "TOUTES COMMUNES" entry which are duplicates of the individual
        # cities entries in Logicimmo module.
        matching_cities = [
            city
            for city in matching_cities
            if not (city.backend == 'logicimmo' and
                    city.name.startswith('TOUTES COMMUNES'))
        ]

141 142 143 144
        # Then, build queries by grouping cities by at most 3
        for cities_batch in tools.batch(matching_cities, 3):
            query = Query()
            query.cities = list(cities_batch)
Phyks's avatar
Phyks committed
145 146 147 148

            try:
                query.house_types = [
                    getattr(
149
                        HOUSE_TYPES,
Phyks's avatar
Phyks committed
150 151 152 153 154 155 156 157 158 159
                        house_type.upper()
                    )
                    for house_type in constraints_dict["house_types"]
                ]
            except AttributeError:
                LOGGER.error("Invalid house types constraint.")
                return None

            try:
                query.type = getattr(
160 161
                    POSTS_TYPES,
                    constraints_dict["type"].upper()
Phyks's avatar
Phyks committed
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
                )
            except AttributeError:
                LOGGER.error("Invalid post type constraint.")
                return None

            query.area_min = constraints_dict["area"][0]
            query.area_max = constraints_dict["area"][1]
            query.cost_min = constraints_dict["cost"][0]
            query.cost_max = constraints_dict["cost"][1]
            query.nb_rooms = constraints_dict["rooms"][0]

            queries.append(query)

        return queries

177
    def query(self, query, max_entries=None, store_personal_data=False):
Phyks's avatar
Phyks committed
178
        """
Phyks's avatar
Phyks committed
179
        Fetch the housings posts matching a given WebOOB query.
Phyks's avatar
Phyks committed
180

Phyks's avatar
Phyks committed
181
        :param query: A WebOOB `weboob.capabilities.housing.Query`` object.
Phyks's avatar
Phyks committed
182
        :param max_entries: Maximum number of entries to fetch.
183
        :param store_personal_data: Whether personal data should be fetched
Phyks's avatar
Phyks committed
184
            from housing posts (phone number etc).
Phyks's avatar
Phyks committed
185 186 187
        :return: The matching housing posts, dumped as a list of JSON objects.
        """
        housings = []
188 189
        # List the useful backends for this specific query
        useful_backends = [x.backend for x in query.cities]
Phyks's avatar
Phyks committed
190
        # TODO: Handle max_entries better
191 192
        try:
            for housing in itertools.islice(
193 194 195 196 197 198 199 200 201
                    self.webnip.do(
                        'search_housings',
                        query,
                        # Only run the call on the required backends.
                        # Otherwise, WebOOB is doing weird stuff and returning
                        # nonsense.
                        backends=[x for x in self.backends
                                  if x.name in useful_backends]
                    ),
202 203
                    max_entries
            ):
204 205
                if not store_personal_data:
                    housing.phone = None
206 207 208 209 210 211 212
                housings.append(json.dumps(housing, cls=WeboobEncoder))
        except CallErrors as exc:
            # If an error occured, just log it
            LOGGER.error(
                "An error occured while fetching the housing posts: %s",
                str(exc)
            )
Phyks's avatar
Phyks committed
213 214
        return housings

215
    def info(self, full_flat_id, store_personal_data=False):
Phyks's avatar
Phyks committed
216 217 218
        """
        Get information (details) about an housing post.

Phyks's avatar
Phyks committed
219
        :param full_flat_id: A WebOOB housing post id, in complete form
Phyks's avatar
Phyks committed
220
            (ID@BACKEND)
221
        :param store_personal_data: Whether personal data should be fetched
Phyks's avatar
Phyks committed
222
            from housing posts (phone number etc).
Phyks's avatar
Phyks committed
223 224 225
        :return: The details in JSON.
        """
        flat_id, backend_name = full_flat_id.rsplit("@", 1)
Phyks's avatar
Phyks committed
226 227 228 229 230 231 232 233 234 235
        try:
            backend = next(
                backend
                for backend in self.backends
                if backend.name == backend_name
            )
        except StopIteration:
            LOGGER.error("Backend %s is not available.", backend_name)
            return "{}"

236 237
        try:
            housing = backend.get_housing(flat_id)
Phyks's avatar
Phyks committed
238 239
            # Otherwise, we miss the @backend afterwards
            housing.id = full_flat_id
240
            if not store_personal_data:
241
                # Ensure phone is cleared
242
                housing.phone = None
243 244 245
            else:
                # Ensure phone is fetched
                backend.fillobj(housing, 'phone')
Phyks's avatar
Phyks committed
246 247

            return json.dumps(housing, cls=WeboobEncoder)
248
        except Exception as exc:  # pylint: disable=broad-except
249 250 251 252 253 254
            # If an error occured, just log it
            LOGGER.error(
                "An error occured while fetching housing %s: %s",
                full_flat_id,
                str(exc)
            )
255
            return "{}"
256

Phyks's avatar
Phyks committed
257

258
def fetch_flats(config):
Phyks's avatar
Phyks committed
259
    """
Phyks's avatar
Phyks committed
260
    Fetch the available flats using the Flatboob / WebOOB config.
Phyks's avatar
Phyks committed
261 262

    :param config: A config dict.
263
    :return: A dict mapping constraint in config to all available matching
Phyks's avatar
Phyks committed
264
        flats.
Phyks's avatar
Phyks committed
265
    """
266 267 268 269
    fetched_flats = {}

    for constraint_name, constraint in config["constraints"].items():
        LOGGER.info("Loading flats for constraint %s...", constraint_name)
Phyks's avatar
Phyks committed
270 271
        with WebOOBProxy(config) as webOOB_proxy:
            queries = webOOB_proxy.build_queries(constraint)
272 273 274
            housing_posts = []
            for query in queries:
                housing_posts.extend(
Phyks's avatar
Phyks committed
275
                    webOOB_proxy.query(query, config["max_entries"],
276
                                       config["store_personal_data"])
277
                )
Phyks's avatar
Phyks committed
278 279
        LOGGER.info("Fetched %d flats.", len(housing_posts))

280
        constraint_flats_list = [json.loads(flat) for flat in housing_posts]
Phyks's avatar
Phyks committed
281
        constraint_flats_list = [WebOOBProxy.restore_decimal_fields(flat)
282 283 284
                                 for flat in constraint_flats_list]
        fetched_flats[constraint_name] = constraint_flats_list
    return fetched_flats
Phyks's avatar
Phyks committed
285 286


Phyks's avatar
Phyks committed
287
def fetch_details(config, flat_id):
Phyks's avatar
Phyks committed
288
    """
Phyks's avatar
Phyks committed
289
    Fetch the additional details for a flat using Flatboob / WebOOB.
Phyks's avatar
Phyks committed
290

Phyks's avatar
Phyks committed
291
    :param config: A config dict.
Phyks's avatar
Phyks committed
292 293 294
    :param flat_id: ID of the flat to fetch details for.
    :return: A flat dict with all the available data.
    """
Phyks's avatar
Phyks committed
295
    with WebOOBProxy(config) as webOOB_proxy:
Phyks's avatar
Phyks committed
296
        LOGGER.info("Loading additional details for flat %s.", flat_id)
Phyks's avatar
Phyks committed
297
        webOOB_output = webOOB_proxy.info(flat_id,
298
                                          config["store_personal_data"])
Phyks's avatar
Phyks committed
299

Phyks's avatar
Phyks committed
300 301
    flat_details = json.loads(webOOB_output)
    flat_details = WebOOBProxy.restore_decimal_fields(flat_details)
Phyks's avatar
Phyks committed
302
    LOGGER.info("Fetched details for flat %s.", flat_id)
Phyks's avatar
Phyks committed
303 304 305 306

    return flat_details


307
def load_flats_from_file(json_file, config):
Phyks's avatar
Phyks committed
308 309 310 311
    """
    Load a dumped flats list from JSON file.

    :param json_file: The file to load housings list from.
312
    :return: A dict mapping constraint in config to all available matching
Phyks's avatar
Phyks committed
313
        flats.
314 315

    .. note::
Phyks's avatar
Phyks committed
316

317 318 319
        As we do not know which constraint is met by a given flat, all the
        flats are returned for any available constraint, and they will be
        filtered out afterwards.
Phyks's avatar
Phyks committed
320 321 322 323 324 325 326 327 328
    """
    flats_list = []
    try:
        LOGGER.info("Loading flats list from file %s", json_file)
        with open(json_file, "r") as fh:
            flats_list = json.load(fh)
        LOGGER.info("Found %d flats.", len(flats_list))
    except (IOError, ValueError):
        LOGGER.error("File %s is not a valid dump file.", json_file)
329 330 331 332
    return {
        constraint_name: flats_list
        for constraint_name in config["constraints"]
    }
333 334


335
def load_flats_from_db(config):
336 337 338 339
    """
    Load flats from database.

    :param config: A config dict.
340
    :return: A dict mapping constraint in config to all available matching
Phyks's avatar
Phyks committed
341
        flats.
342
    """
343
    get_session = database.init_db(config["database"], config["search_index"])
344

345
    loaded_flats = collections.defaultdict(list)
346
    with get_session() as session:
347 348 349
        for flat in session.query(flat_model.Flat).all():
            loaded_flats[flat.flatisfy_constraint].append(flat.json_api_repr())
    return loaded_flats