Skip to content

Commit

Permalink
#29 Support var len features
Browse files Browse the repository at this point in the history
  • Loading branch information
@oaksharks
oaksharks committed Nov 25, 2020
1 parent afc8849 commit 1ed27c8
Show file tree
Hide file tree
Showing 11 changed files with 664 additions and 41 deletions.
7 changes: 7 additions & 0 deletions deeptables/datasets/dsutils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,10 @@ def load_bank():
data = pd.read_csv(f'{basedir}/bank-uci.csv')
logger.info(f'data shape:{data.shape}')
return data


def load_movielens():
logger.info(f'Base dir:{os.getcwd()}')
data = pd.read_csv(f'{basedir}/movielens_sample.txt')
logger.info(f'data shape:{data.shape}')
return data
201 changes: 201 additions & 0 deletions deeptables/datasets/movielens_sample.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
user_id,movie_id,rating,timestamp,title,genres,gender,age,occupation,zip
3299,235,4,968035345,Ed Wood (1994),Comedy|Drama,F,25,4,19119
3630,3256,3,966536874,Patriot Games (1992),Action|Thriller,M,18,4,77005
517,105,4,976203603,"Bridges of Madison County, The (1995)",Drama|Romance,F,25,14,55408
785,2115,3,975430389,Indiana Jones and the Temple of Doom (1984),Action|Adventure,M,18,19,29307
5848,909,5,957782527,"Apartment, The (1960)",Comedy|Drama,M,50,20,20009
2996,2799,1,972769867,Problem Child 2 (1991),Comedy,M,18,0,63011
3087,837,5,969738869,Matilda (1996),Children's|Comedy,F,1,1,90802
872,3092,5,975273310,Chushingura (1962),Drama,M,50,1,20815
4094,529,5,966223349,Searching for Bobby Fischer (1993),Drama,M,25,17,49017
1868,3508,3,974694703,"Outlaw Josey Wales, The (1976)",Western,M,50,11,92346
2913,1387,5,971769808,Jaws (1975),Action|Horror,F,35,20,98119
380,3481,5,976316283,High Fidelity (2000),Comedy,M,25,2,92024
2073,1784,5,974759084,As Good As It Gets (1997),Comedy|Drama,F,18,4,13148
80,2059,3,977788576,"Parent Trap, The (1998)",Children's|Drama,M,56,1,49327
3679,2557,1,976298130,I Stand Alone (Seul contre tous) (1998),Drama,M,25,4,68108
2077,788,3,980013556,"Nutty Professor, The (1996)",Comedy|Fantasy|Romance|Sci-Fi,M,18,0,55112
6036,2085,4,956716684,101 Dalmatians (1961),Animation|Children's,F,25,15,32603
3675,532,3,966363610,Serial Mom (1994),Comedy|Crime|Horror,M,35,7,06680
4566,3683,4,964489599,Blood Simple (1984),Drama|Film-Noir,M,35,17,19473
2996,3763,3,972413564,F/X (1986),Action|Crime|Thriller,M,18,0,63011
5831,2458,1,957898337,Armed and Dangerous (1986),Comedy|Crime,M,25,1,92120
1869,1244,2,974695654,Manhattan (1979),Comedy|Drama|Romance,M,45,14,95148
5389,2657,3,960328279,"Rocky Horror Picture Show, The (1975)",Comedy|Horror|Musical|Sci-Fi,M,45,7,01905
1391,1535,3,974851275,Love! Valour! Compassion! (1997),Drama|Romance,M,35,15,20723
3123,2407,3,969324381,Cocoon (1985),Comedy|Sci-Fi,M,25,2,90401
4694,159,3,963602574,Clockers (1995),Drama,M,56,7,40505
1680,1988,3,974709821,Hello Mary Lou: Prom Night II (1987),Horror,M,25,20,95380
2002,1945,4,974677761,On the Waterfront (1954),Crime|Drama,F,56,13,02136-1522
3430,2690,4,979949863,"Ideal Husband, An (1999)",Comedy,F,45,1,15208
425,471,4,976284972,"Hudsucker Proxy, The (1994)",Comedy|Romance,M,25,12,55303
1841,2289,2,974699637,"Player, The (1992)",Comedy|Drama,M,18,0,95037
4964,2348,4,962619587,Sid and Nancy (1986),Drama,M,35,0,94110
4520,2160,4,964883648,Rosemary's Baby (1968),Horror|Thriller,M,25,4,45810
1265,2396,4,1011716691,Shakespeare in Love (1998),Comedy|Romance,F,18,20,49321
2496,1278,5,974435324,Young Frankenstein (1974),Comedy|Horror,M,50,1,37932
5511,2174,4,959787754,Beetlejuice (1988),Comedy|Fantasy,M,45,1,92407
621,833,1,975799925,High School High (1996),Comedy,M,18,4,93560
3045,2762,5,970189524,"Sixth Sense, The (1999)",Thriller,M,45,1,90631
2050,2546,4,975522689,"Deep End of the Ocean, The (1999)",Drama,F,35,3,99504
613,32,4,975812238,Twelve Monkeys (1995),Drama|Sci-Fi,M,35,20,10562
366,1077,5,978471241,Sleeper (1973),Comedy|Sci-Fi,M,50,15,55126
5108,367,4,962338215,"Mask, The (1994)",Comedy|Crime|Fantasy,F,25,9,93940
4502,1960,4,965094644,"Last Emperor, The (1987)",Drama|War,M,50,0,01379
5512,1801,5,959713840,"Man in the Iron Mask, The (1998)",Action|Drama|Romance,F,25,17,01701
1861,2642,2,974699627,Superman III (1983),Action|Adventure|Sci-Fi,M,50,16,92129
1667,1240,4,975016698,"Terminator, The (1984)",Action|Sci-Fi|Thriller,M,50,16,98516
753,434,3,975460449,Cliffhanger (1993),Action|Adventure|Crime,M,1,10,42754
1836,2736,5,974826228,Brighton Beach Memoirs (1986),Comedy,M,25,0,10016
5626,474,5,959052158,In the Line of Fire (1993),Action|Thriller,M,56,16,32043
1601,1396,4,978576948,Sneakers (1992),Crime|Drama|Sci-Fi,M,25,12,83001
4725,1100,4,963369546,Days of Thunder (1990),Action|Romance,M,35,5,96707-1321
2837,2396,5,972571456,Shakespeare in Love (1998),Comedy|Romance,M,18,0,49506
1776,3882,4,1001558470,Bring It On (2000),Comedy,M,25,0,45801
2820,457,2,972662398,"Fugitive, The (1993)",Action|Thriller,F,35,0,02138
1834,2288,3,1038179198,"Thing, The (1982)",Action|Horror|Sci-Fi|Thriller,M,35,5,10990
284,2716,4,976570902,Ghostbusters (1984),Comedy|Horror,M,25,12,91910
2744,588,1,973215985,Aladdin (1992),Animation|Children's|Comedy|Musical,M,18,17,53818
881,4,2,975264028,Waiting to Exhale (1995),Comedy|Drama,M,18,14,76401
2211,916,3,974607067,Roman Holiday (1953),Comedy|Romance,M,45,6,01950
2271,2671,4,1007158806,Notting Hill (1999),Comedy|Romance,M,50,14,13210
1010,2953,1,975222613,Home Alone 2: Lost in New York (1992),Children's|Comedy,M,25,0,10310
1589,2594,4,974735454,Open Your Eyes (Abre los ojos) (1997),Drama|Romance|Sci-Fi,M,25,0,95136
1724,597,5,976441106,Pretty Woman (1990),Comedy|Romance,M,18,4,00961
2590,2097,3,973840056,Something Wicked This Way Comes (1983),Children's|Horror,M,18,4,94044
1717,1352,3,1009256707,Albino Alligator (1996),Crime|Thriller,F,50,6,30307
1391,3160,2,974850796,Magnolia (1999),Drama,M,35,15,20723
1941,1263,3,974954220,"Deer Hunter, The (1978)",Drama|War,M,35,17,94550
3526,2867,4,966906064,Fright Night (1985),Comedy|Horror,M,35,2,62263-3004
5767,198,3,958192148,Strange Days (1995),Action|Crime|Sci-Fi,M,25,2,75287
5355,590,4,960596927,Dances with Wolves (1990),Adventure|Drama|Western,M,56,0,78232
5788,156,4,958108785,Blue in the Face (1995),Comedy,M,25,0,92646
1078,1307,4,974938851,When Harry Met Sally... (1989),Comedy|Romance,F,45,9,95661
3808,61,2,965973222,Eye for an Eye (1996),Drama|Thriller,M,25,7,60010
974,3897,4,975106398,Almost Famous (2000),Comedy|Drama,M,35,19,94930
5153,1290,4,961972292,Some Kind of Wonderful (1987),Drama|Romance,M,25,7,60046
5732,2115,3,958434069,Indiana Jones and the Temple of Doom (1984),Action|Adventure,F,25,11,02111
4627,2478,3,964110136,Three Amigos! (1986),Comedy|Western,M,56,1,45224
1884,1831,2,975648062,Lost in Space (1998),Action|Sci-Fi|Thriller,M,45,20,93108
4284,517,4,965277546,Rising Sun (1993),Action|Drama|Mystery,M,50,7,40601
1383,468,2,975979732,"Englishman Who Went Up a Hill, But Came Down a Mountain, The (1995)",Comedy|Romance,F,25,7,19806
2230,2873,3,974599097,Lulu on the Bridge (1998),Drama|Mystery|Romance,F,45,1,60302
2533,2266,4,974055724,"Butcher's Wife, The (1991)",Comedy|Romance,F,25,3,49423
6040,3224,5,956716750,Woman in the Dunes (Suna no onna) (1964),Drama,M,25,6,11106
4384,2918,5,965171739,Ferris Bueller's Day Off (1986),Comedy,M,25,0,43623
5156,3688,3,961946487,Porky's (1981),Comedy,M,18,14,10024
615,296,3,975805801,Pulp Fiction (1994),Crime|Drama,M,50,17,32951
2753,3045,3,973198964,Peter's Friends (1992),Comedy|Drama,F,50,20,27516
2438,1125,5,974259943,"Return of the Pink Panther, The (1974)",Comedy,M,35,1,22903
5746,1242,4,958354460,Glory (1989),Action|Drama|War,M,18,15,94061
5157,3462,5,961944604,Modern Times (1936),Comedy,M,35,1,74012
3402,1252,5,967433929,Chinatown (1974),Film-Noir|Mystery|Thriller,M,35,20,30306
76,593,5,977847255,"Silence of the Lambs, The (1991)",Drama|Thriller,M,35,7,55413
2067,1019,3,974658834,"20,000 Leagues Under the Sea (1954)",Adventure|Children's|Fantasy|Sci-Fi,M,50,16,06430
2181,2020,3,979353437,Dangerous Liaisons (1988),Drama|Romance,M,25,0,45245
3947,593,5,965691680,"Silence of the Lambs, The (1991)",Drama|Thriller,M,25,0,90019
546,218,4,976069421,Boys on the Side (1995),Comedy|Drama,F,25,0,37211
1246,3030,5,1032056405,Yojimbo (1961),Comedy|Drama|Western,M,18,4,98225
4214,3186,5,965319143,"Girl, Interrupted (1999)",Drama,F,25,0,20121
2841,680,3,982805796,Alphaville (1965),Sci-Fi,M,50,12,98056
4205,3175,4,965321085,Galaxy Quest (1999),Adventure|Comedy|Sci-Fi,F,25,15,87801
1120,1097,4,974911354,E.T. the Extra-Terrestrial (1982),Children's|Drama|Fantasy|Sci-Fi,M,18,4,95616
5371,3194,3,960481000,"Way We Were, The (1973)",Drama,M,25,11,55408
2695,1278,5,973310827,Young Frankenstein (1974),Comedy|Horror,M,35,11,46033
3312,520,2,976673070,Robin Hood: Men in Tights (1993),Comedy,F,18,4,90039
5039,1792,1,962513044,U.S. Marshalls (1998),Action|Thriller,F,35,4,97068
4655,2146,3,963903103,St. Elmo's Fire (1985),Drama|Romance,F,25,1,92037
3558,1580,5,966802528,Men in Black (1997),Action|Adventure|Comedy|Sci-Fi,M,18,17,66044
506,3354,1,976208080,Mission to Mars (2000),Sci-Fi,M,25,16,55103-1006
3568,1230,3,966745594,Annie Hall (1977),Comedy|Romance,M,25,0,98503
2943,1197,5,971319983,"Princess Bride, The (1987)",Action|Adventure|Comedy|Romance,M,35,12,95864
716,737,3,982881364,Barb Wire (1996),Action|Sci-Fi,M,18,4,98188
5964,454,3,956999469,"Firm, The (1993)",Drama|Thriller,M,18,5,97202
4802,1208,4,996034747,Apocalypse Now (1979),Drama|War,M,56,1,40601
1106,3624,4,974920622,Shanghai Noon (2000),Action,M,18,4,90241
3410,2565,3,967419652,"King and I, The (1956)",Musical,M,35,1,20653
1273,3095,5,974814536,"Grapes of Wrath, The (1940)",Drama,M,35,2,19123
1706,1916,4,974709448,Buffalo 66 (1998),Action|Comedy|Drama,M,25,20,19134
4889,590,5,962909224,Dances with Wolves (1990),Adventure|Drama|Western,M,18,4,63108
4966,2100,3,962609782,Splash (1984),Comedy|Fantasy|Romance,M,50,14,55407
4238,1884,4,965343416,Fear and Loathing in Las Vegas (1998),Comedy|Drama,M,35,16,44691
5365,1042,3,960502974,That Thing You Do! (1996),Comedy,M,18,12,90250
415,1302,3,977501743,Field of Dreams (1989),Drama,F,35,0,55406
4658,1009,5,963966553,Escape to Witch Mountain (1975),Adventure|Children's|Fantasy,M,25,4,99163
854,345,3,975357801,"Adventures of Priscilla, Queen of the Desert, The (1994)",Comedy|Drama,F,25,16,44092
2857,436,4,972509362,Color of Night (1994),Drama|Thriller,M,25,0,10469
1835,1330,4,974878241,April Fool's Day (1986),Comedy|Horror,M,25,19,11501
1321,2240,3,974778494,My Bodyguard (1980),Drama,F,25,14,34639
3274,3698,2,979767184,"Running Man, The (1987)",Action|Adventure|Sci-Fi,M,25,20,02062
5893,2144,3,957470619,Sixteen Candles (1984),Comedy,M,25,7,02139
3436,2724,3,967328026,Runaway Bride (1999),Comedy|Romance,M,35,0,98503
3315,2918,5,967942960,Ferris Bueller's Day Off (1986),Comedy,M,25,12,78731
5056,2700,5,962488280,"South Park: Bigger, Longer and Uncut (1999)",Animation|Comedy,M,45,1,16673
5256,208,2,961271616,Waterworld (1995),Action|Adventure,M,25,16,30269
4290,1193,4,965274348,One Flew Over the Cuckoo's Nest (1975),Drama,M,25,17,98661
1010,1379,2,975220259,Young Guns II (1990),Action|Comedy|Western,M,25,0,10310
829,904,4,975368038,Rear Window (1954),Mystery|Thriller,M,1,19,53711
5953,480,4,957143581,Jurassic Park (1993),Action|Adventure|Sci-Fi,M,1,10,21030
4732,3016,4,963332896,Creepshow (1982),Horror,M,25,14,24450
4815,3181,5,972240802,Titus (1999),Drama,F,50,18,04849
1164,1894,2,1004486985,Six Days Seven Nights (1998),Adventure|Comedy|Romance,F,25,19,90020
4373,3167,5,965180829,Carnal Knowledge (1971),Drama,M,50,12,32920
5293,1374,4,961055887,Star Trek: The Wrath of Khan (1982),Action|Adventure|Sci-Fi,M,25,12,95030
1579,3101,4,981272057,Fatal Attraction (1987),Thriller,M,25,0,60201
2600,3147,5,973804787,"Green Mile, The (1999)",Drama|Thriller,M,25,14,19312
1283,480,4,974793389,Jurassic Park (1993),Action|Adventure|Sci-Fi,F,18,1,94607
3242,3062,5,968341175,"Longest Day, The (1962)",Action|Drama|War,M,50,13,94089
3618,3374,3,967116272,Daughters of the Dust (1992),Drama,M,56,17,22657
3762,1337,4,966434517,"Body Snatcher, The (1945)",Horror,M,50,6,11746
1015,1184,3,975018699,Mediterraneo (1991),Comedy|War,M,35,3,11220
4645,2344,5,963976808,Runaway Train (1985),Action|Adventure|Drama|Thriller,F,50,6,48094
3184,1397,4,968709039,Bastard Out of Carolina (1996),Drama,F,25,18,21214
1285,1794,4,974833328,Love and Death on Long Island (1997),Comedy|Drama,M,35,4,98125
5521,3354,2,959833154,Mission to Mars (2000),Sci-Fi,F,25,6,02118
1472,2278,3,974767792,Ronin (1998),Action|Crime|Thriller,M,25,7,90248
5630,21,4,980085414,Get Shorty (1995),Action|Comedy|Drama,M,35,17,06854
3710,3033,5,966272980,Spaceballs (1987),Comedy|Sci-Fi,M,1,10,02818
192,761,1,977028390,"Phantom, The (1996)",Adventure,M,18,1,10977
1285,1198,5,974880310,Raiders of the Lost Ark (1981),Action|Adventure,M,35,4,98125
2174,1046,4,974613044,Beautiful Thing (1996),Drama|Romance,M,50,12,87505
635,1270,4,975768106,Back to the Future (1985),Comedy|Sci-Fi,M,56,17,33785
910,412,5,975207742,"Age of Innocence, The (1993)",Drama,F,50,0,98226
1752,2021,4,975729332,Dune (1984),Fantasy|Sci-Fi,M,25,3,96813
1408,198,4,974762924,Strange Days (1995),Action|Crime|Sci-Fi,M,25,0,90046
4738,1242,4,963279051,Glory (1989),Action|Drama|War,M,56,1,23608
1503,1971,2,974748897,"Nightmare on Elm Street 4: The Dream Master, A (1988)",Horror,M,25,12,92688
3053,1296,3,970601837,"Room with a View, A (1986)",Drama|Romance,F,25,3,55102
3471,3614,2,973297828,Honeymoon in Vegas (1992),Comedy|Romance,M,18,4,80302
678,1972,3,988638700,"Nightmare on Elm Street 5: The Dream Child, A (1989)",Horror,M,25,0,34952
3483,2561,3,986327282,True Crime (1999),Crime|Thriller,F,45,7,30260
3910,3108,5,965756244,"Fisher King, The (1991)",Comedy|Drama|Romance,M,25,20,91505
182,1089,1,977085647,Reservoir Dogs (1992),Crime|Thriller,M,18,4,03052
1755,1653,3,1036917836,Gattaca (1997),Drama|Sci-Fi|Thriller,F,18,4,77005
3589,70,2,966658567,From Dusk Till Dawn (1996),Action|Comedy|Crime|Horror|Thriller,F,45,0,80010
471,3481,4,976222483,High Fidelity (2000),Comedy,M,35,7,08904
1141,813,2,974878678,Larger Than Life (1996),Comedy,F,25,3,84770
5227,1196,2,961476022,Star Wars: Episode V - The Empire Strikes Back (1980),Action|Adventure|Drama|Sci-Fi|War,M,18,10,64050
1303,2344,2,974837844,Runaway Train (1985),Action|Adventure|Drama|Thriller,M,25,19,94111
5080,3102,5,962412804,Jagged Edge (1985),Thriller,F,50,12,95472
2023,1012,4,1006290836,Old Yeller (1957),Children's|Drama,M,18,4,56001
3759,2151,5,966094413,"Gods Must Be Crazy II, The (1989)",Comedy,M,35,6,54751
1685,2664,2,974709721,Invasion of the Body Snatchers (1956),Horror|Sci-Fi,M,35,12,95833
4715,1221,4,963508830,"Godfather: Part II, The (1974)",Action|Crime|Drama,M,25,2,97205
1591,350,5,974742941,"Client, The (1994)",Drama|Mystery|Thriller,M,50,7,26501
4227,3635,3,965411938,"Spy Who Loved Me, The (1977)",Action,M,25,19,11414-2520
1908,36,5,974697744,Dead Man Walking (1995),Drama,M,56,13,95129
5365,1892,4,960503255,"Perfect Murder, A (1998)",Mystery|Thriller,M,18,12,90250
1579,2420,4,981272235,"Karate Kid, The (1984)",Drama,M,25,0,60201
1866,3948,5,974753321,Meet the Parents (2000),Comedy,M,25,7,94043
4238,3543,4,965415533,Diner (1982),Comedy|Drama,M,35,16,44691
3590,2000,5,966657892,Lethal Weapon (1987),Action|Comedy|Crime|Drama,F,18,15,02115
3401,3256,5,980115327,Patriot Games (1992),Action|Thriller,M,35,7,76109
3705,540,2,966287116,Sliver (1993),Thriller,M,45,7,30076
4973,1246,3,962607149,Dead Poets Society (1989),Drama,F,56,2,949702
4947,380,4,962651180,True Lies (1994),Action|Adventure|Comedy|Romance,M,35,17,90035
2346,1416,4,974413811,Evita (1996),Drama|Musical,F,1,10,48105
1427,3596,3,974840560,Screwed (2000),Comedy,M,25,12,21401
3868,1626,3,965855033,Fire Down Below (1997),Action|Drama|Thriller,M,18,12,73112
249,2369,3,976730191,Desperately Seeking Susan (1985),Comedy|Romance,F,18,14,48126
5720,349,4,958503395,Clear and Present Danger (1994),Action|Adventure|Thriller,M,25,0,60610
877,1485,3,975270899,Liar Liar (1997),Comedy,M,25,0,90631
17 changes: 17 additions & 0 deletions deeptables/models/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class ModelConfig(collections.namedtuple('ModelConfig',
'earlystopping_patience',
'gpu_usage_strategy',
'distribute_strategy',
'var_len_categorical_columns',
])):
def __hash__(self):
return self.name.__hash__()
Expand Down Expand Up @@ -126,7 +127,22 @@ def __new__(cls,
earlystopping_patience=1,
gpu_usage_strategy=consts.GPU_USAGE_STRATEGY_GROWTH,
distribute_strategy=None,
var_len_categorical_columns=None, # a tuple3, format is (column_name, separator, pool_strategy), pool_strategy is one of max,avg; e.g. [('genres', '|', 'avg' )]
):

if var_len_categorical_columns is not None and len(var_len_categorical_columns) > 0:
# check items
for v in var_len_categorical_columns:
_name = v[0]
if not isinstance(v, (tuple, list)) or len(v) != 3:
raise ValueError("Var len column config should be a tuple 3.")
if exclude_columns is not None:
if _name in exclude_columns:
raise ValueError(f"Var len column {_name} can not put in 'exclude_columns' ")
if categorical_columns is not None and isinstance(categorical_columns, list):
if _name in categorical_columns:
raise ValueError(f"Var len column {_name} can not put in 'categorical_columns' ")

nets = deepnets.get_nets(nets)

if home_dir is None and os.environ.get(consts.ENV_DEEPTABLES_HOME) is not None:
Expand Down Expand Up @@ -175,6 +191,7 @@ def __new__(cls,
earlystopping_patience,
gpu_usage_strategy,
distribute_strategy,
var_len_categorical_columns,
)
@property
def first_metric_name(self):
Expand Down
Loading

0 comments on commit 1ed27c8

Please sign in to comment.