Source code for examples.taskqueue

"""This simple example shows how to use the `DataRetriever` class to retrieve
information about some links.

It uses the Celery task system in the background.

Please ensure that you have a worker running already or that you will set
`config.WORKER_START = True` in your config file.
"""
from loguru import logger

from magpie import init, shutdown
from magpie.data.testdata import simple_folder
from magpie.fetch import DataRetriever
from magpie.task.queue import ping
from magpie.util import pformat




def main():
    """
    Sends retrieval tasks to the task queue.
    Note: A magpie worker needs to be active. This is done either by starting a worker manually with `just worker`
    or by setting `WORKER_START = True` in `config.py`
    """
    init()

    workers = ping(timeout=5)
    logger.info(f'hello! here\'s the status of the worker pool: {workers}')

    magpie = DataRetriever()
    data = simple_folder()
    data.remove('Rust')
    data['Python'].remove('Best practices')
    magpie.identify(data)
    magpie.fetch(data)
    magpie.wait_for_tasks_completion()

    logger.info(data.as_text())
    logger.info(pformat(data))

    shutdown()



if __name__ == '__main__':
    main()