unicorn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0d345d26-29ea-4c65-8c01-040bcf9237d0",
   "metadata": {},
   "source": [
    "# Unicorn Log Breakdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dff7c116-1224-4b49-8452-604848f88074",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pathlib\n",
    "cwd = pathlib.Path.cwd()\n",
    "\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "#bokeh\n",
    "import pandas_bokeh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb03c73e-9c6e-4fb0-b156-3a5d7662fd25",
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_file = cwd / 'unicorn6.csv'\n",
    "df = pd.read_csv(csv_file,  dtype={\"line_number\": int, \"line_length\": int, \"status\": str})\n",
    "\n",
    "df['rate_limit'] = df['rate_limit'].fillna(-1)\n",
    "df['integration_id'] = df['integration_id'].fillna(-1)\n",
    "df['date'] = pd.to_datetime(df['now'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19153ebe-3767-48b3-a12a-3a68b50b254f",
   "metadata": {},
   "source": [
    "### User Agent Top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfaf831f-76b9-4bf2-befb-ef04f6a00514",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['user_agent'].value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "784b86ad-570b-4064-ab50-3cb6ba28eda5",
   "metadata": {},
   "source": [
    "### User Top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8d51a19-707f-4a12-bb21-78caa599d6d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['user'].value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99e9db5d-d841-4e2c-a52f-0d2bce6c3186",
   "metadata": {},
   "source": [
    "### Repositories Top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd7bda32-e553-4c2a-879a-ff4a32244e47",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['repo'].value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5c40dab-2b63-45bc-8eca-d6aac6c1b48d",
   "metadata": {},
   "source": [
    "### Routes Top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2736864-d299-4863-9ba0-ca20d9d8f56a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['route'].value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcf66715-3268-4b48-b86c-cea288a259e5",
   "metadata": {},
   "source": [
    "### Status Top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01484fb7-c471-4882-8ede-06456b271e69",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['status'].value_counts().head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "455780ec-53c2-49c8-a4df-e06d101e6899",
   "metadata": {},
   "source": [
    "### Path info Top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b4b792f-2612-4663-bb9f-db5fb5693675",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['path_info'].value_counts().head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97f37dac-846f-4194-bfee-27755e2da500",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ad3a080-a9a2-4ad2-aa0d-87573de97457",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the sample frequency to something useful for your case\n",
    "sample_frequency = '120S'\n",
    "buckets = df.groupby([pd.Grouper(key='date', axis=0, freq=sample_frequency),'status'])['status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e25cf6b2-412d-443e-995b-4052549d53d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "buckets.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c1535be-71a5-4723-b560-567de770c49e",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = buckets.size().unstack()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "527b35b4-8c52-4055-89f5-6bf2758005a3",
   "metadata": {},
   "source": [
    "### Plot the status column over the sample period"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3b1dfa5-cfb8-4d26-ac53-9f8e8d3d9d90",
   "metadata": {},
   "outputs": [],
   "source": [
    "n.plot(figsize=(24,18))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bf96a05-b0f5-4247-a8c4-3d9779d2fd57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# seaborn plot\n",
    "sns.relplot(data=n, height=6, aspect=15/8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0d86b16-fbc2-4bcf-88ea-c1ef71dd140e",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(25,8))\n",
    "sns.lineplot(data=n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d333be4b-d32b-4123-9429-380200877bd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# bokeh plot\n",
    "n.plot_bokeh(kind=\"line\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8946d38a-f2b0-47d1-827e-9d366426f7af",
   "metadata": {},
   "source": [
    "### Create CSV files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e2977b9-fc23-4a73-8d97-b156cc2132dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "n.to_csv('unicorn-status-breakdown.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8dcdd18-083b-47f3-a36d-8beca441acd0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}