Skip to content

Commit

Permalink
revert batching salesforce updates (#988)
Browse files Browse the repository at this point in the history 
* revert batching salesforce updates

* update routine spec

* update vcrs
  • Loading branch information
@mwvolo
mwvolo authored Sep 7, 2021
1 parent 2cbba23 commit 70fb08a
Show file tree
Hide file tree
Showing 15 changed files with 596 additions and 587 deletions.
222 changes: 115 additions & 107 deletions app/routines/update_user_salesforce_info.rb
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
class UpdateUserSalesforceInfo
BATCH_SIZE = 250

COLLEGE_TYPES = [
'College/University (4)',
'Technical/Community College (2)',
'Career School/For-Profit (2)'
'College/University (4)',
'Technical/Community College (2)',
'Career School/For-Profit (2)'
]
HIGH_SCHOOL_TYPES = [ 'High School' ]
K12_TYPES = [ 'K-12 School' ]
Expand All @@ -14,10 +12,10 @@ class UpdateUserSalesforceInfo
FOREIGN_SCHOOL_LOCATIONS = [ 'Foreign' ]

ADOPTION_STATUSES = {
"Current Adopter" => true,
"Future Adopter" => true,
"Past Adopter" => false,
"Not Adopter" => false
"Current Adopter" => true,
"Future Adopter" => true,
"Past Adopter" => false,
"Not Adopter" => false
}

def initialize(allow_error_email:)
Expand All @@ -41,7 +39,7 @@ def call
prepare_contacts

schools_by_salesforce_id = School.select(:id, :salesforce_id).where(
salesforce_id: @contacts_by_id.values.compact.map(&:school_id)
salesforce_id: @contacts_by_id.values.compact.map(&:school_id)
).index_by(&:salesforce_id)

# Go through all users that have already have a Salesforce Contact ID and make sure
Expand All @@ -54,7 +52,6 @@ def call
cache_contact_and_school_data_in_user!(contact, school, user)
rescue StandardError => ee
error!(exception: ee, user: user)
Sentry.capture_exception ee
end
end

Expand Down Expand Up @@ -90,7 +87,6 @@ def call
end
rescue StandardError => ee
error!(exception: ee, user: user)
Sentry.capture_exception ee
end
end
end
Expand Down Expand Up @@ -128,17 +124,16 @@ def call

unless user.is_newflow? # because the new Accounts flow works differently; don't mess with it.
user.faculty_status =
if statuses == ["Converted"]
:rejected_faculty
else
:pending_faculty
end
if statuses == ["Converted"]
:rejected_faculty
else
:pending_faculty
end
end

user.save! if user.changed?
rescue StandardError => ee
error!(exception: ee, user: user)
Sentry.capture_exception ee
end
end
end
Expand All @@ -156,14 +151,41 @@ def call
self
end

def contacts
# The query below is not particularly fast, takes around a minute. We could
# try to do something fancier, like only query contacts modified in the last day
# or keep track of when the SF data was last updated and use those timestamps
# to limit what data we pull from Salesforce (could have a global field in redis
# or could copy SF contact "LastModifiedAt" to a "sf_refreshed_at" field on each
# User record).
#
# Here's one example query as a starting point:
# ...Contact.order("LastModifiedDate").where("LastModifiedDate >= #{1.day.ago.utc.iso8601}")
#
# TODO: Need to move the duplicated email detection code to SF before we change anything
# If we use timestamps to limit results returned, need to be very careful
# to avoid race conditions such as missing records that were modified
# while we were querying SF
# If we don't use timestamps, should load the contacts in chunks of 1,000 or 10,000
# Or maybe try https://github.com/gooddata/salesforce_bulk_query

@contacts ||= OpenStax::Salesforce::Remote::Contact
.select(
:id, :email, :email_alt, :faculty_verified,
:school_type, :adoption_status, :grant_tutor_access
)
.includes(:school)
.to_a
end

def leads
# Leads come from many sources; we only care about those created for faculty
# verification ("OSC Faculty")

@leads ||= OpenStax::Salesforce::Remote::Lead
.where(source: "OSC Faculty")
.select(:id, :email)
.to_a
.where(source: "OSC Faculty")
.select(:id, :email)
.to_a
end

def prepare_leads
Expand All @@ -181,60 +203,46 @@ def prepare_leads
end

def prepare_contacts
last_id = nil
begin
loop do
sf_contacts = OpenStax::Salesforce::Remote::Contact.select(:id, :email, :email_alt, :faculty_verified,:school_type, :adoption_status, :grant_tutor_access).includes(:school).order(:id).limit(BATCH_SIZE)
sf_contacts = sf_contacts.where("id > '#{last_id}'") unless last_id.nil?
sf_contacts = sf_contacts.to_a
last_id = sf_contacts.last.id unless sf_contacts.last.nil?

colliding_emails = []

# Store each contact in our internal maps; keep track of which contacts have
# colliding emails so we can clear them out below (don't want to clear out
# until all contacts have been examined so that we don't miss a collision)

sf_contacts.each do |contact|
emails = [contact.email, contact.email_alt].compact
.map(&:downcase)
.map(&:strip)
.uniq

emails.each do |email|
if (colliding_contact = @contacts_by_email[email])
colliding_emails.push(email)
error!(
message: "#{email} is listed on contact #{contact.id} and contact #{colliding_contact.id}" \
"; neither contact will be synched to an OpenStax Account until this is resolved."
)
else
@contacts_by_email[email] = contact
@contacts_by_id[contact.id] = contact
end
end
colliding_emails = []

# Store each contact in our internal maps; keep track of which contacts have
# colliding emails so we can clear them out below (don't want to clear out
# until all contacts have been examined so that we don't miss a collision)

contacts.each do |contact|
emails = [contact.email, contact.email_alt].compact
.map(&:downcase)
.map(&:strip)
.uniq

emails.each do |email|
if (colliding_contact = @contacts_by_email[email])
colliding_emails.push(email)
error!(
message: "#{email} is listed on contact #{contact.id} and contact #{colliding_contact.id}" \
"; neither contact will be synched to an OpenStax Account until this is resolved."
)
else
@contacts_by_email[email] = contact
@contacts_by_id[contact.id] = contact
end
end
end

# Go through colliding emails and clear their Contacts out of our hashes so that
# we don't assign these Contacts to users until a human resolves the collisions
# Go through colliding emails and clear their Contacts out of our hashes so that
# we don't assign these Contacts to users until a human resolves the collisions

colliding_emails.uniq.each do |colliding_email|
contact = @contacts_by_email[colliding_email]
@contacts_by_id[contact.id] = nil
@contacts_by_email[colliding_email] = nil
end
break if sf_contacts.length < BATCH_SIZE
end
rescue StandardError => ee
error!(exception: ee)
Sentry.capture_exception ee
colliding_emails.uniq.each do |colliding_email|
contact = @contacts_by_email[colliding_email]
@contacts_by_id[contact.id] = nil
@contacts_by_email[colliding_email] = nil
end
end

def cache_contact_and_school_data_in_user!(contact, school, user)
if contact.nil?
warn(
"User #{user.id} previously linked to contact #{user.salesforce_contact_id} but that" \
"User #{user.id} previously linked to contact #{user.salesforce_contact_id} but that" \
" contact is no longer present; resetting user's contact ID, faculty status, school type, and school location"
)
user.salesforce_contact_id = nil
Expand All @@ -245,44 +253,44 @@ def cache_contact_and_school_data_in_user!(contact, school, user)
user.salesforce_contact_id = contact.id

user.faculty_status = case contact.faculty_verified
when "Confirmed"
:confirmed_faculty
when "Pending"
:pending_faculty
when /Rejected/
:rejected_faculty
when NilClass
:no_faculty_info
else
raise "Unknown faculty_verified field: '#{
contact.faculty_verified}'' on contact #{contact.id}"
when "Confirmed"
:confirmed_faculty
when "Pending"
:pending_faculty
when /Rejected/
:rejected_faculty
when NilClass
:no_faculty_info
else
raise "Unknown faculty_verified field: '#{
contact.faculty_verified}'' on contact #{contact.id}"
end

# TODO: We can read school_type and school_location from the cached School records instead,
# but better wait 1 additional release to let the Schools be cached and linked
user.school_type = case contact.school_type
when *COLLEGE_TYPES
:college
when *HIGH_SCHOOL_TYPES
:high_school
when *K12_TYPES
:k12_school
when *HOME_SCHOOL_TYPES
:home_school
when NilClass
:unknown_school_type
else
:other_school_type
when *COLLEGE_TYPES
:college
when *HIGH_SCHOOL_TYPES
:high_school
when *K12_TYPES
:k12_school
when *HOME_SCHOOL_TYPES
:home_school
when NilClass
:unknown_school_type
else
:other_school_type
end

sf_school = contact.school
user.school_location = case sf_school&.school_location
when *DOMESTIC_SCHOOL_LOCATIONS
:domestic_school
when *FOREIGN_SCHOOL_LOCATIONS
:foreign_school
else
:unknown_school_location
when *DOMESTIC_SCHOOL_LOCATIONS
:domestic_school
when *FOREIGN_SCHOOL_LOCATIONS
:foreign_school
else
:unknown_school_location
end

unless contact.adoption_status.blank?
Expand All @@ -304,19 +312,19 @@ def cache_contact_and_school_data_in_user!(contact, school, user)
let_sf_know_to_send_fac_ver_email = true

SecurityLog.create!(
user: user,
application: nil,
remote_ip: nil,
event_type: :faculty_verified,
event_data: { user_id: user.id, salesforce_contact_id: contact.id }
user: user,
application: nil,
remote_ip: nil,
event_type: :faculty_verified,
event_data: { user_id: user.id, salesforce_contact_id: contact.id }
)
end

user.save! if user.changed?

if let_sf_know_to_send_fac_ver_email
contact.update_attributes!(
send_faculty_verification_to: user.guessed_preferred_confirmed_email
send_faculty_verification_to: user.guessed_preferred_confirmed_email
)
end
end
Expand All @@ -326,9 +334,9 @@ def error!(exception: nil, message: nil, user: nil)

error[:message] = message || exception.try(:message)
error[:exception] = {
class: exception.class.name,
message: exception.message,
first_backtrace_line: exception.backtrace.try(:first)
class: exception.class.name,
message: exception.message,
first_backtrace_line: exception.backtrace.try(:first)
} if exception.present?
error[:user] = user.id if user.present?

Expand All @@ -349,9 +357,9 @@ def notify_errors

if @allow_error_email && Settings::Salesforce.user_info_error_emails_enabled
DevMailer.inspect_object(
object: @errors,
subject: "(#{Rails.application.secrets.environment_name}) UpdateUserSalesforceInfo errors",
to: Rails.application.secrets.salesforce[:mail_recipients]
object: @errors,
subject: "(#{Rails.application.secrets.environment_name}) UpdateUserSalesforceInfo errors",
to: Rails.application.secrets.salesforce[:mail_recipients]
).deliver_later
end
end
Expand Down
Loading

0 comments on commit 70fb08a

Please sign in to comment.