Add DataFrames.jl extension

Project.toml

@@ -9,32 +9,36 @@ MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
 MutableArithmetics = "d8a4904e-b15c-11e9-3269-09a3773c0cb0"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [weakdeps]
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
 
 [extensions]
+JuMPDataFramesExt = "DataFrames"
 JuMPDimensionalDataExt = "DimensionalData"
 
 [compat]
+DataFrames = "1"
 DimensionalData = "0.24, 0.25, 0.26.2, 0.27"
 LinearAlgebra = "<0.0.1, 1.6"
 MacroTools = "0.5"
 MathOptInterface = "1.25.2"
 MutableArithmetics = "1.1"
 OrderedCollections = "1"
-Printf = "<0.0.1, 1.6"
 PrecompileTools = "1"
+Printf = "<0.0.1, 1.6"
 SparseArrays = "<0.0.1, 1.6"
 Test = "<0.0.1, 1.6"
 julia = "1.6"
 
 [extras]
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["DimensionalData", "Test"]
+test = ["DataFrames", "DimensionalData", "Test"]

docs/src/tutorials/linear/multi.jl

@@ -3,7 +3,7 @@
 # v.2.0. If a copy of the MPL was not distributed with this file, You can       #src
 # obtain one at https://mozilla.org/MPL/2.0/.                                   #src
 
-# # The multi-commodity flow problem
+# # Working with SQLite and DataFrames
 
 # **This tutorial was originally contributed by Louis Luangkesorn.**
 
@@ -23,10 +23,10 @@ using JuMP
 import DataFrames
 import HiGHS
 import SQLite
+import SQLite: DBInterface
 import Tables
 import Test
 
-const DBInterface = SQLite.DBInterface
 
 # ## Formulation
 
@@ -119,38 +119,62 @@ products =
 
 model = Model(HiGHS.Optimizer)
 set_silent(model)
-@variable(model, x[origins, destinations, products] >= 0)
+@variable(
+    model,
+    x[origin in origins, destination in destinations, product in products] >= 0,
+    container = DataFrames.DataFrame,
+)
 
 # One approach when working with databases is to extract all of the data into a
-# Julia datastructure. For example, let's pull the cost table into a DataFrame
-# and then construct our objective by iterating over the rows of the DataFrame:
+# Julia datastructure. For example, let's pull the cost table into a DataFrame:
 
 cost = DBInterface.execute(db, "SELECT * FROM cost") |> DataFrames.DataFrame
-@objective(
-    model,
-    Max,
-    sum(r.cost * x[r.origin, r.destination, r.product] for r in eachrow(cost)),
-);
 
-# If we don't want to use a DataFrame, we can use a `Tables.rowtable` instead:
+# and then join the decision variables:
 
-supply = DBInterface.execute(db, "SELECT * FROM supply") |> Tables.rowtable
-for r in supply
-    @constraint(model, sum(x[r.origin, :, r.product]) <= r.supply)
+function natural_join(left, right)
+    on_names = intersect(names(left), names(right))
+    return DataFrames.innerjoin(left, right; on = on_names)
 end
 
-# Another approach is to execute the query, and then to iterate through the rows
-# of the query using `Tables.rows`:
+cost_x = natural_join(cost, x)
+
+# We've defined a new function, `natural_join`, to simplify the process of
+# joining two DataFrames. This fuction acts like the `NATURAL JOIN` statment in
+# SQL.
+
+# Our objective is the inner product of two columns:
 
-demand = DBInterface.execute(db, "SELECT * FROM demand")
-for r in Tables.rows(demand)
-    @constraint(model, sum(x[:, r.destination, r.product]) == r.demand)
+@objective(model, Max, cost_x.cost' * cost_x.value);
+
+# The supply constraint is more complicated. A useful utility is a function that
+# sums the `.value` column after grouping on a set of columns:
+
+function sum_value_by(df, cols)
+    gdf = DataFrames.groupby(df, cols)
+    return DataFrames.combine(gdf, :value => sum => :value)
 end
 
-# !!! warning
-#     Iterating through the rows of a query result works by incrementing a
-#     cursor inside the database. As a consequence, you cannot call
-#     `Tables.rows` twice on the same query result.
+# Here is it in action:
+
+sum_value_by(x, [:origin, :product])
+
+# The constraint that the supply must be less than or equal to a capacity can
+# now be written as:
+
+supply = natural_join(
+    DBInterface.execute(db, "SELECT * FROM supply") |> DataFrames.DataFrame,
+    sum_value_by(x, [:origin, :product]),
+)
+@constraint(model, supply.value .<= supply.supply);
+
+# The demand constraint ca be written similarly:
+
+demand = natural_join(
+    DBInterface.execute(db, "SELECT * FROM demand") |> DataFrames.DataFrame,
+    sum_value_by(x, [:destination, :product]),
+)
+@constraint(model, demand.value .== demand.demand);
 
 # The SQLite queries can be arbitrarily complex. For example, here's a query
 # which builds every possible origin-destination pair:
@@ -164,13 +188,12 @@ od_pairs = DBInterface.execute(
     INNER JOIN locations b
     ON a.type = 'origin' AND b.type = 'destination'
     """,
-)
+) |> DataFrames.DataFrame
 
 # With a constraint that we cannot send more than 625 units between each pair:
 
-for r in Tables.rows(od_pairs)
-    @constraint(model, sum(x[r.origin, r.destination, :]) <= 625)
-end
+od = natural_join(od_pairs, sum_value_by(x, [:origin, :destination]))
+@constraint(model, od.value .<= 625);
 
 # ## Solution
 
@@ -181,12 +204,7 @@ Test.@test is_solved_and_feasible(model)
 Test.@test objective_value(model) == 225_700.0      #src
 solution_summary(model)
 
-# and print the solution:
+# and obtain the solution:
 
-begin
-    println("         ", join(products, ' '))
-    for o in origins, d in destinations
-        v = lpad.([round(Int, value(x[o, d, p])) for p in products], 5)
-        println(o, " ", d, " ", join(replace.(v, "   0" => "  . "), " "))
-    end
-end
+x.value = value.(x.value)
+x[x.value .> 0, :]

ext/JuMPDataFramesExt.jl

@@ -0,0 +1,23 @@
+#  Copyright 2017, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+module JuMPDataFramesExt
+
+import DataFrames
+import JuMP
+
+function JuMP.Containers.container(
+    f::Function,
+    indices,
+    ::Type{DataFrames.DataFrame},
+    names::AbstractVector,
+)
+    rows = vec(collect(indices))
+    df = DataFrames.DataFrame(NamedTuple{tuple(names...)}(arg) for arg in rows)
+    df.value = [f(arg...) for arg in rows]
+    return df
+end
+
+end #module

ext/test_DataFrames.jl

@@ -0,0 +1,83 @@
+#  Copyright 2017, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
+#  This Source Code Form is subject to the terms of the Mozilla Public
+#  License, v. 2.0. If a copy of the MPL was not distributed with this
+#  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+module TestContainersDataFrames
+
+using Test
+
+using DataFrames
+using JuMP
+
+function test_dimension_data_vector()
+    model = Model()
+    @variable(model, x[i = 2:4], container = DataFrame)
+    @test x isa DataFrame
+    @test size(x) == (3, 2)
+    @test names(x) == ["i", "value"]
+    return
+end
+
+function test_dimension_data_matrix()
+    model = Model()
+    @variable(model, x[i = 2:4, j = ["a", "b"]], container = DataFrame)
+    @test x isa DataFrame
+    @test size(x) == (6, 3)
+    @test names(x) == ["i", "j", "value"]
+    @test sum(x[x.j .== "a", :value]) isa AffExpr
+    return
+end
+
+function test_dimension_data_triangle()
+    model = Model()
+    @variable(model, x[i = 2:4, j in i:4], container = DataFrame)
+    @test x isa DataFrame
+    @test size(x) == (6, 3)
+    @test names(x) == ["i", "j", "value"]
+    return
+end
+
+function test_dimension_data_sparse()
+    model = Model()
+    @variable(model, x[i in 1:4, j in 1:4; isodd(i + j)], container = DataFrame)
+    @test x isa DataFrame
+    @test size(x) == (8, 3)
+    @test x.i == [1, 1, 2, 2, 3, 3, 4, 4]
+    @test x.j == [2, 4, 1, 3, 2, 4, 1, 3]
+    @test names(x) == ["i", "j", "value"]
+    return
+end
+
+function test_dataframes_expression()
+    model = Model()
+    B = ["a", "b"]
+    @variable(model, x[i = 2:4, j = B], container = DataFrame)
+    @expression(
+        model,
+        expr[j = B],
+        sum(x[x.j .== j, :value]),
+        container = DataFrame,
+    )
+    @test expr isa DataFrame
+    @test expr.j == ["a", "b"]
+    expr2 = DataFrames.combine(
+        DataFrames.groupby(x, :j),
+        :value => sum => :value,
+    )
+    @test expr == expr2
+    return
+end
+
+function test_data_frames_missing_names()
+    model = Model()
+    x = @variable(model, [1:3, 1:2], container = DataFrame)
+    @test all(startswith.(names(x), ["##", "##", "value"]))
+    x = @variable(model, [i in 1:3, 1:2], container = DataFrame)
+    @test all(startswith.(names(x), ["i", "##", "value"]))
+    x = @variable(model, [1:3, j in 1:2], container = DataFrame)
+    @test all(startswith.(names(x), ["##", "j", "value"]))
+    return
+end
+
+end