Repository

Shows how to store data based on configs.

import dman
from dman.numeric import carray, barray
from dman.plotting import PdfFigure

import numpy as np
import matplotlib.pyplot as plt
import warnings


@dman.modelclass(compact=True, storable=True)
class Config:
    degrees: carray[int]

    def key(self):
        return f"fit({','.join([str(d) for d in self.degrees])})"


@dman.modelclass(storable=True, store_by_field=True)
class Experiment:
    x: barray = dman.recordfield(subdir='samples')
    y: barray = dman.recordfield(subdir='samples')
    z: dman.mruns
    fig: PdfFigure = None


def experiment(cfg: Config):
    """Generate experiment data.
    Based on: https://numpy.org/doc/stable/reference/generated/numpy.polyfit.html.
    """
    xp = np.linspace(-2, 6, 100)
    x = np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0])
    y = np.array([0.0, 0.8, 0.9, 0.1, -0.8, -1.0])
    z = dman.mruns(stem="poly", store_subdir=False, subdir='fit')

    fig, ax = plt.subplots(1, 1)
    ax.plot(x, y, ".", color="gray", label="samples")
    for d in dman.tui.progress(cfg.degrees, description="order"):
        with warnings.catch_warnings():
            warnings.simplefilter("ignore", np.RankWarning)
            zz = np.polyfit(x, y, d)
        z.append(zz.view(barray))
        p = np.poly1d(zz)
        ax.plot(xp, p(xp), "-", label=f"fit deg={d}")
    ax.legend()
    return Experiment(x, y, z, PdfFigure(fig))


def evaluate(cfg: Config):
    """Evaluate the provided configuration.
    If the experiment was executed before we return its result
    otherwise a new experiment is executed."""
    with dman.track("poly", default_factory=dman.mdict_factory(store_subdir=True, store_by_key=True)) as reg:
        reg: dman.mdict = reg  # for type hinting
        value = reg.get(cfg.key(), None)
        if value is None:
            value = experiment(cfg)
            reg[cfg.key()] = value
        return value

# Clear the old runs if they exist (for testing)
dman.clean('poly')

# Execute several configurations
res1 = evaluate(Config([1, 5, 7]))
res2 = evaluate(Config([8, 3]))

• 
• 

order [3/3] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00
order [2/2] ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00

We can examine the results and see that they are correct.

dman.tui.pprint(res1)
dman.tui.pprint(res2)

Experiment(
│   z=[barray([-0.30285714,  0.75714286]), barray([-8.33333333e-03,  1.25000000e-01, -5.75000000e-01,
│   │    6.25000000e-01,  6.33333333e-01,  1.40925649e-14]), barray([-1.91046771e-04,  8.51553918e-04,  5.63990582e-03,
│   │   -3.21702635e-03, -1.74163595e-01,  9.60491579e-02,
│   │    8.75031051e-01, -7.25194643e-16])],
│   fig=<dman.plotting.PdfFigure object at 0x7fe4a1cd9870>
)
Experiment(
│   z=[barray([-2.90550072e-05,  5.43231076e-05,  7.81064119e-04,
│   │    2.85322742e-03, -7.22529560e-03, -1.19892134e-01,
│   │   -6.49029671e-03,  9.29948167e-01,  4.35116786e-15]), barray([ 0.08703704, -0.81349206,  1.69312169, -0.03968254])],
│   fig=<dman.plotting.PdfFigure object at 0x7fe4a1bf7d30>
)

We can re-execute an old configuration

res3 = evaluate(Config([1, 5, 7]))
dman.tui.pprint(res3)  # data is unloaded since it was loaded from disk

Experiment(
│   z=[UL[_num__barray], UL[_num__barray], UL[_num__barray]],
│   fig=<dman.plotting.PdfFigure object at 0x7fe4a2cd9cf0>
)

The file structure is then as follows. Note that a pdf has been created for each experiment containing the produced figures. Feel free to take a look.

dman.tui.walk_directory(dman.mount("poly"))

📂 .dman/cache/examples:patterns:example1_repository/poly
┣━━ 📂 fit(1,5,7)
┃   ┣━━ 📂 fit
┃   ┃   ┣━━ 📄 poly-0.npy (144 bytes)
┃   ┃   ┣━━ 📄 poly-1.npy (176 bytes)
┃   ┃   ┗━━ 📄 poly-2.npy (192 bytes)
┃   ┣━━ 📂 samples
┃   ┃   ┣━━ 📄 x.npy (176 bytes)
┃   ┃   ┗━━ 📄 y.npy (176 bytes)
┃   ┣━━ 📄 fig.pdf (14.3 kB)
┃   ┗━━ 📄 fit(1,5,7).json (1.5 kB)
┣━━ 📂 fit(8,3)
┃   ┣━━ 📂 fit
┃   ┃   ┣━━ 📄 poly-0.npy (200 bytes)
┃   ┃   ┗━━ 📄 poly-1.npy (160 bytes)
┃   ┣━━ 📂 samples
┃   ┃   ┣━━ 📄 x.npy (176 bytes)
┃   ┃   ┗━━ 📄 y.npy (176 bytes)
┃   ┣━━ 📄 fig.pdf (13.3 kB)
┃   ┗━━ 📄 fit(8,3).json (1.3 kB)
┗━━ 📄 poly.json (641 bytes)